Data Adapter Architecture

Sightline uses a pluggable adapter pattern to support multiple data sources. This document explains the architecture and how to create custom adapters.

Overview

┌─────────────────────────────────────────────────────────────┐
│                      DataContext                             │
│  ┌─────────────────────────────────────────────────────┐    │
│  │           Adapter Factory (createAdapter)            │    │
│  │  ┌─────────┐  ┌─────────┐  ┌─────────┐            │    │
│  │  │ Static  │  │Supabase │  │   API   │  ...       │    │
│  │  │ Adapter │  │ Adapter │  │ Adapter │            │    │
│  │  └────┬────┘  └────┬────┘  └────┬────┘            │    │
│  └───────│────────────│────────────│─────────────────┘    │
│          │            │            │                       │
│          ▼            ▼            ▼                       │
│     ┌─────────────────────────────────────────┐           │
│     │           Unified Incident[]             │           │
│     └─────────────────────────────────────────┘           │
└─────────────────────────────────────────────────────────────┘

Adapters are created dynamically via a factory function based on the data source configuration. Each fetch operation creates fresh adapter instances.

Adapter Interface

All adapters implement the DataAdapter interface:

// src/adapters/types.ts
export interface DataAdapter {
  /** Unique identifier */
  readonly id: string;
  
  /** Human-readable name */
  readonly name: string;
  
  /** Fetch all incidents, optionally filtered */
  getIncidents(filters?: IncidentFilters): Promise<Incident[]>;
  
  /** Fetch a single incident by ID */
  getIncidentById(id: string): Promise<Incident | null>;
  
  /** Get available filter options from this source */
  getFilterOptions(): Promise<FilterOptions>;
  
  /** Clear cached data (optional) */
  clearCache?(): void;
}

export interface FilterOptions {
  countries: string[];
  siteTypes: string[];
  locationSensitivities: LocationSensitivity[];
  incidentTypes: IncidentType[];
}

Built-in Adapters

Static Adapter

Loads data from JSON files in the public/ directory. Implements caching with configurable TTL.

// src/adapters/static-adapter.ts (simplified - see source for full implementation)
export class StaticAdapter implements DataAdapter {
  readonly id: string;
  readonly name: string;
  private path: string;
  private cache: AdapterCache<Incident[]>;
  
  constructor(config: DataSourceConfig) {
    this.id = config.id;
    this.name = config.name;
    this.path = config.adapterConfig.path as string;
    this.cache = new AdapterCache({ ttlMs: config.adapterConfig.cacheTtlMs });
  }
  
  async getIncidents(filters?: IncidentFilters): Promise<Incident[]> {
    const cacheKey = 'incidents';
    
    // Return cached data if available and not expired
    const cached = this.cache.get(cacheKey);
    if (cached) return applyIncidentFilters(cached, filters);
    
    // Fetch and parse JSON
    const response = await fetch(this.path);
    const data = await response.json();
    
    // Handle both array and { incidents: [...] } formats
    const rawIncidents = Array.isArray(data) ? data : data.incidents || [];
    
    // Validate, cache, and return
    const { valid } = validateIncidentArray(rawIncidents);
    this.cache.set(cacheKey, valid);
    return applyIncidentFilters(valid, filters);
  }
  
  async getIncidentById(id: string): Promise<Incident | null> {
    const incidents = await this.getIncidents();
    return incidents.find(i => i.id === id) || null;
  }
  
  clearCache(): void {
    this.cache.clear();
  }
}

Supabase Adapter

Connects to a Supabase PostgreSQL database using RPC functions. Fetches all published incidents and applies filters client-side for flexibility.

// src/adapters/supabase-adapter.ts (simplified - see source for full implementation)
export class SupabaseAdapter implements DataAdapter {
  readonly id: string;
  readonly name: string;
  private cache: AdapterCache<Incident[]>;
  
  constructor(config: DataSourceConfig) {
    this.id = config.id;
    this.name = config.name;
    this.cache = new AdapterCache({ ttlMs: config.adapterConfig.cacheTtlMs });
  }
  
  async getIncidents(filters?: IncidentFilters): Promise<Incident[]> {
    const cacheKey = 'incidents';
    
    // Check cache first
    const cached = this.cache.get(cacheKey);
    if (cached) return applyIncidentFilters(cached, filters);
    
    // Fetch via RPC (uses direct fetch to Supabase REST API)
    const rows = await this.rpc(SUPABASE_RPC.GET_ALL_INCIDENTS, {
      p_limit: QUERY_LIMITS.ALL_INCIDENTS,
      p_status: 'published',
    });
    
    // Transform rows to Incident format
    const incidents = rows.map(rowToIncident);
    this.cache.set(cacheKey, incidents);
    
    // Apply filters client-side
    return applyIncidentFilters(incidents, filters);
  }
  
  clearCache(): void {
    this.cache.clear();
  }
}

Note: The Supabase adapter uses RPC functions for data retrieval, with filtering applied client-side. This allows for flexible filtering without requiring database changes.

API Adapter

Fetches from external REST APIs. Converts filters to query parameters for server-side filtering when supported.

// src/adapters/api-adapter.ts
export class ApiAdapter implements DataAdapter {
  readonly id: string;
  readonly name: string;
  private endpoint: string;
  private headers: Record<string, string>;
  private cache: AdapterCache<Incident[]>;
  
  constructor(config: DataSourceConfig) {
    this.id = config.id;
    this.name = config.name;
    this.endpoint = config.adapterConfig.endpoint as string;
    this.cache = new AdapterCache({ ttlMs: config.adapterConfig.cacheTtlMs });
    
    this.headers = {
      'Content-Type': 'application/json',
      ...(config.adapterConfig.apiKey && {
        'Authorization': `Bearer ${config.adapterConfig.apiKey}`
      }),
      ...config.adapterConfig.headers,
    };
  }
  
  async getIncidents(filters?: IncidentFilters): Promise<Incident[]> {
    // Build URL with filter query params
    const url = new URL(this.endpoint);
    if (filters) {
      // Convert filters to query params (inline)
      if (filters.countries?.length) {
        url.searchParams.set('countries', filters.countries.join(','));
      }
      // ... other filter params
    }
    
    const response = await fetch(url.toString(), {
      headers: this.headers,
    });
    
    if (!response.ok) {
      throw new Error(`API request failed: ${response.status}`);
    }
    
    const data = await response.json();
    // Handle both array and { incidents: [...] } formats
    const incidents = Array.isArray(data) ? data : data.incidents;
    return incidents;
  }
  
  // ... other methods
}

Adapter Factory

Adapters are created via a factory function:

// src/adapters/index.ts
export function createAdapter(config: DataSourceConfig): DataAdapter {
  switch (config.adapter) {
    case 'static':
      return new StaticAdapter(config);
    case 'supabase':
      return new SupabaseAdapter(config);
    case 'api':
      return new ApiAdapter(config);
    default:
      throw new Error(`Unknown adapter type: ${config.adapter}`);
  }
}

Creating Custom Adapters

Step 1: Implement the Interface

// src/adapters/my-adapter.ts
import type { DataAdapter, Incident, DataSourceConfig, FilterOptions } from './types';

export class MyAdapter implements DataAdapter {
  readonly id: string;
  readonly name: string;
  private config: MyAdapterConfig;
  
  constructor(config: DataSourceConfig) {
    this.id = config.id;
    this.name = config.name;
    this.config = config.adapterConfig as MyAdapterConfig;
  }
  
  async getIncidents(): Promise<Incident[]> {
    // Your data fetching logic
    const rawData = await this.fetchFromMySource();
    
    // Transform to Incident format
    return rawData.map(this.transformToIncident);
  }
  
  async getIncidentById(id: string): Promise<Incident | null> {
    const incidents = await this.getIncidents();
    return incidents.find(i => i.id === id) || null;
  }
  
  async getFilterOptions(): Promise<FilterOptions> {
    const incidents = await this.getIncidents();
    return {
      countries: [...new Set(incidents.map(i => i.location?.country))],
      siteTypes: [...new Set(incidents.map(i => i.location?.siteType))],
      // ... extract other options
    };
  }
  
  private async fetchFromMySource(): Promise<RawData[]> {
    // Implementation
  }
  
  private transformToIncident(raw: RawData): Incident {
    return {
      id: raw.id,
      status: 'published',
      createdAt: raw.created,
      updatedAt: raw.modified,
      temporal: {
        date: raw.date,
        dateCertainty: 'exact',
      },
      location: {
        name: raw.location,
        country: raw.country,
        latitude: raw.lat,
        longitude: raw.lng,
        siteType: mapSiteType(raw.type),
        locationSensitivity: 'standard',
      },
      // ... map other fields
    };
  }
}

Step 2: Define Config Type

interface MyAdapterConfig {
  connectionString: string;
  apiVersion?: string;
  // ... your config options
}

Step 3: Register in Factory

// src/adapters/index.ts
import { MyAdapter } from './my-adapter';

export function createAdapter(config: DataSourceConfig): DataAdapter {
  switch (config.adapter) {
    // ... existing cases
    case 'my-adapter':
      return new MyAdapter(config);
  }
}

Step 4: Use in Config

// map.config.ts
dataSources: [
  {
    id: 'my-source',
    name: 'My Data Source',
    adapter: 'my-adapter',
    adapterConfig: {
      connectionString: 'my://connection',
      apiVersion: 'v2',
    },
    enabled: true,
  },
]

Error Handling

Adapters should handle errors gracefully:

async getIncidents(): Promise<Incident[]> {
  try {
    const response = await fetch(this.endpoint);
    
    if (!response.ok) {
      throw new AdapterError(
        `HTTP ${response.status}`,
        this.id,
        'FETCH_FAILED'
      );
    }
    
    return await response.json();
  } catch (error) {
    if (error instanceof AdapterError) throw error;
    
    // Wrap unknown errors
    throw new AdapterError(
      error.message,
      this.id,
      'UNKNOWN_ERROR'
    );
  }
}

Caching Strategy

Adapters can implement caching:

class CachedAdapter implements DataAdapter {
  private cache: Map<string, { data: Incident[], timestamp: number }> = new Map();
  private ttl = 5 * 60 * 1000; // 5 minutes
  
  async getIncidents(): Promise<Incident[]> {
    const cached = this.cache.get('incidents');
    
    if (cached && Date.now() - cached.timestamp < this.ttl) {
      return cached.data;
    }
    
    const fresh = await this.fetchFresh();
    this.cache.set('incidents', { data: fresh, timestamp: Date.now() });
    return fresh;
  }
  
  invalidateCache() {
    this.cache.clear();
  }
}

Testing Adapters

// src/adapters/__tests__/my-adapter.test.ts
describe('MyAdapter', () => {
  it('fetches incidents', async () => {
    const adapter = new MyAdapter({
      id: 'test',
      name: 'Test',
      adapter: 'my-adapter',
      adapterConfig: { /* mock config */ },
      enabled: true,
    });
    
    const incidents = await adapter.getIncidents();
    
    expect(incidents).toBeInstanceOf(Array);
    expect(incidents[0]).toMatchObject({
      id: expect.any(String),
      temporal: expect.any(Object),
      location: expect.any(Object),
    });
  });
  
  it('handles errors gracefully', async () => {
    // Mock a failing request
    const adapter = new MyAdapter({ /* ... */ });
    
    await expect(adapter.getIncidents()).rejects.toThrow(AdapterError);
  });
});

Data Adapter Architecture

Sightline uses a pluggable adapter pattern to support multiple data sources. This document explains the architecture and how to create custom adapters.

Overview

┌─────────────────────────────────────────────────────────────┐
│                      DataContext                             │
│  ┌─────────────────────────────────────────────────────┐    │
│  │           Adapter Factory (createAdapter)            │    │
│  │  ┌─────────┐  ┌─────────┐  ┌─────────┐            │    │
│  │  │ Static  │  │Supabase │  │   API   │  ...       │    │
│  │  │ Adapter │  │ Adapter │  │ Adapter │            │    │
│  │  └────┬────┘  └────┬────┘  └────┬────┘            │    │
│  └───────│────────────│────────────│─────────────────┘    │
│          │            │            │                       │
│          ▼            ▼            ▼                       │
│     ┌─────────────────────────────────────────┐           │
│     │           Unified Incident[]             │           │
│     └─────────────────────────────────────────┘           │
└─────────────────────────────────────────────────────────────┘

Adapters are created dynamically via a factory function based on the data source configuration. Each fetch operation creates fresh adapter instances.

Adapter Interface

All adapters implement the DataAdapter interface:

// src/adapters/types.ts
export interface DataAdapter {
  /** Unique identifier */
  readonly id: string;
  
  /** Human-readable name */
  readonly name: string;
  
  /** Fetch all incidents, optionally filtered */
  getIncidents(filters?: IncidentFilters): Promise<Incident[]>;
  
  /** Fetch a single incident by ID */
  getIncidentById(id: string): Promise<Incident | null>;
  
  /** Get available filter options from this source */
  getFilterOptions(): Promise<FilterOptions>;
  
  /** Clear cached data (optional) */
  clearCache?(): void;
}

export interface FilterOptions {
  countries: string[];
  siteTypes: string[];
  locationSensitivities: LocationSensitivity[];
  incidentTypes: IncidentType[];
}

Built-in Adapters

Static Adapter

Loads data from JSON files in the public/ directory. Implements caching with configurable TTL.

// src/adapters/static-adapter.ts (simplified - see source for full implementation)
export class StaticAdapter implements DataAdapter {
  readonly id: string;
  readonly name: string;
  private path: string;
  private cache: AdapterCache<Incident[]>;
  
  constructor(config: DataSourceConfig) {
    this.id = config.id;
    this.name = config.name;
    this.path = config.adapterConfig.path as string;
    this.cache = new AdapterCache({ ttlMs: config.adapterConfig.cacheTtlMs });
  }
  
  async getIncidents(filters?: IncidentFilters): Promise<Incident[]> {
    const cacheKey = 'incidents';
    
    // Return cached data if available and not expired
    const cached = this.cache.get(cacheKey);
    if (cached) return applyIncidentFilters(cached, filters);
    
    // Fetch and parse JSON
    const response = await fetch(this.path);
    const data = await response.json();
    
    // Handle both array and { incidents: [...] } formats
    const rawIncidents = Array.isArray(data) ? data : data.incidents || [];
    
    // Validate, cache, and return
    const { valid } = validateIncidentArray(rawIncidents);
    this.cache.set(cacheKey, valid);
    return applyIncidentFilters(valid, filters);
  }
  
  async getIncidentById(id: string): Promise<Incident | null> {
    const incidents = await this.getIncidents();
    return incidents.find(i => i.id === id) || null;
  }
  
  clearCache(): void {
    this.cache.clear();
  }
}

Supabase Adapter

Connects to a Supabase PostgreSQL database using RPC functions. Fetches all published incidents and applies filters client-side for flexibility.

// src/adapters/supabase-adapter.ts (simplified - see source for full implementation)
export class SupabaseAdapter implements DataAdapter {
  readonly id: string;
  readonly name: string;
  private cache: AdapterCache<Incident[]>;
  
  constructor(config: DataSourceConfig) {
    this.id = config.id;
    this.name = config.name;
    this.cache = new AdapterCache({ ttlMs: config.adapterConfig.cacheTtlMs });
  }
  
  async getIncidents(filters?: IncidentFilters): Promise<Incident[]> {
    const cacheKey = 'incidents';
    
    // Check cache first
    const cached = this.cache.get(cacheKey);
    if (cached) return applyIncidentFilters(cached, filters);
    
    // Fetch via RPC (uses direct fetch to Supabase REST API)
    const rows = await this.rpc(SUPABASE_RPC.GET_ALL_INCIDENTS, {
      p_limit: QUERY_LIMITS.ALL_INCIDENTS,
      p_status: 'published',
    });
    
    // Transform rows to Incident format
    const incidents = rows.map(rowToIncident);
    this.cache.set(cacheKey, incidents);
    
    // Apply filters client-side
    return applyIncidentFilters(incidents, filters);
  }
  
  clearCache(): void {
    this.cache.clear();
  }
}

Note: The Supabase adapter uses RPC functions for data retrieval, with filtering applied client-side. This allows for flexible filtering without requiring database changes.

API Adapter

Fetches from external REST APIs. Converts filters to query parameters for server-side filtering when supported.

// src/adapters/api-adapter.ts
export class ApiAdapter implements DataAdapter {
  readonly id: string;
  readonly name: string;
  private endpoint: string;
  private headers: Record<string, string>;
  private cache: AdapterCache<Incident[]>;
  
  constructor(config: DataSourceConfig) {
    this.id = config.id;
    this.name = config.name;
    this.endpoint = config.adapterConfig.endpoint as string;
    this.cache = new AdapterCache({ ttlMs: config.adapterConfig.cacheTtlMs });
    
    this.headers = {
      'Content-Type': 'application/json',
      ...(config.adapterConfig.apiKey && {
        'Authorization': `Bearer ${config.adapterConfig.apiKey}`
      }),
      ...config.adapterConfig.headers,
    };
  }
  
  async getIncidents(filters?: IncidentFilters): Promise<Incident[]> {
    // Build URL with filter query params
    const url = new URL(this.endpoint);
    if (filters) {
      // Convert filters to query params (inline)
      if (filters.countries?.length) {
        url.searchParams.set('countries', filters.countries.join(','));
      }
      // ... other filter params
    }
    
    const response = await fetch(url.toString(), {
      headers: this.headers,
    });
    
    if (!response.ok) {
      throw new Error(`API request failed: ${response.status}`);
    }
    
    const data = await response.json();
    // Handle both array and { incidents: [...] } formats
    const incidents = Array.isArray(data) ? data : data.incidents;
    return incidents;
  }
  
  // ... other methods
}

Adapter Factory

Adapters are created via a factory function:

// src/adapters/index.ts
export function createAdapter(config: DataSourceConfig): DataAdapter {
  switch (config.adapter) {
    case 'static':
      return new StaticAdapter(config);
    case 'supabase':
      return new SupabaseAdapter(config);
    case 'api':
      return new ApiAdapter(config);
    default:
      throw new Error(`Unknown adapter type: ${config.adapter}`);
  }
}

Creating Custom Adapters

Step 1: Implement the Interface

// src/adapters/my-adapter.ts
import type { DataAdapter, Incident, DataSourceConfig, FilterOptions } from './types';

export class MyAdapter implements DataAdapter {
  readonly id: string;
  readonly name: string;
  private config: MyAdapterConfig;
  
  constructor(config: DataSourceConfig) {
    this.id = config.id;
    this.name = config.name;
    this.config = config.adapterConfig as MyAdapterConfig;
  }
  
  async getIncidents(): Promise<Incident[]> {
    // Your data fetching logic
    const rawData = await this.fetchFromMySource();
    
    // Transform to Incident format
    return rawData.map(this.transformToIncident);
  }
  
  async getIncidentById(id: string): Promise<Incident | null> {
    const incidents = await this.getIncidents();
    return incidents.find(i => i.id === id) || null;
  }
  
  async getFilterOptions(): Promise<FilterOptions> {
    const incidents = await this.getIncidents();
    return {
      countries: [...new Set(incidents.map(i => i.location?.country))],
      siteTypes: [...new Set(incidents.map(i => i.location?.siteType))],
      // ... extract other options
    };
  }
  
  private async fetchFromMySource(): Promise<RawData[]> {
    // Implementation
  }
  
  private transformToIncident(raw: RawData): Incident {
    return {
      id: raw.id,
      status: 'published',
      createdAt: raw.created,
      updatedAt: raw.modified,
      temporal: {
        date: raw.date,
        dateCertainty: 'exact',
      },
      location: {
        name: raw.location,
        country: raw.country,
        latitude: raw.lat,
        longitude: raw.lng,
        siteType: mapSiteType(raw.type),
        locationSensitivity: 'standard',
      },
      // ... map other fields
    };
  }
}

Step 2: Define Config Type

interface MyAdapterConfig {
  connectionString: string;
  apiVersion?: string;
  // ... your config options
}

Step 3: Register in Factory

// src/adapters/index.ts
import { MyAdapter } from './my-adapter';

export function createAdapter(config: DataSourceConfig): DataAdapter {
  switch (config.adapter) {
    // ... existing cases
    case 'my-adapter':
      return new MyAdapter(config);
  }
}

Step 4: Use in Config

// map.config.ts
dataSources: [
  {
    id: 'my-source',
    name: 'My Data Source',
    adapter: 'my-adapter',
    adapterConfig: {
      connectionString: 'my://connection',
      apiVersion: 'v2',
    },
    enabled: true,
  },
]

Error Handling

Adapters should handle errors gracefully:

async getIncidents(): Promise<Incident[]> {
  try {
    const response = await fetch(this.endpoint);
    
    if (!response.ok) {
      throw new AdapterError(
        `HTTP ${response.status}`,
        this.id,
        'FETCH_FAILED'
      );
    }
    
    return await response.json();
  } catch (error) {
    if (error instanceof AdapterError) throw error;
    
    // Wrap unknown errors
    throw new AdapterError(
      error.message,
      this.id,
      'UNKNOWN_ERROR'
    );
  }
}

Caching Strategy

Adapters can implement caching:

class CachedAdapter implements DataAdapter {
  private cache: Map<string, { data: Incident[], timestamp: number }> = new Map();
  private ttl = 5 * 60 * 1000; // 5 minutes
  
  async getIncidents(): Promise<Incident[]> {
    const cached = this.cache.get('incidents');
    
    if (cached && Date.now() - cached.timestamp < this.ttl) {
      return cached.data;
    }
    
    const fresh = await this.fetchFresh();
    this.cache.set('incidents', { data: fresh, timestamp: Date.now() });
    return fresh;
  }
  
  invalidateCache() {
    this.cache.clear();
  }
}

Testing Adapters

// src/adapters/__tests__/my-adapter.test.ts
describe('MyAdapter', () => {
  it('fetches incidents', async () => {
    const adapter = new MyAdapter({
      id: 'test',
      name: 'Test',
      adapter: 'my-adapter',
      adapterConfig: { /* mock config */ },
      enabled: true,
    });
    
    const incidents = await adapter.getIncidents();
    
    expect(incidents).toBeInstanceOf(Array);
    expect(incidents[0]).toMatchObject({
      id: expect.any(String),
      temporal: expect.any(Object),
      location: expect.any(Object),
    });
  });
  
  it('handles errors gracefully', async () => {
    // Mock a failing request
    const adapter = new MyAdapter({ /* ... */ });
    
    await expect(adapter.getIncidents()).rejects.toThrow(AdapterError);
  });
});

Data Adapter Architecture

Overview

Adapter Interface

Built-in Adapters

Static Adapter

Supabase Adapter

API Adapter

Adapter Factory

Creating Custom Adapters

Step 1: Implement the Interface

Step 2: Define Config Type

Step 3: Register in Factory

Step 4: Use in Config

Error Handling

Caching Strategy

Testing Adapters

On this page

Data Adapter Architecture

Overview

Adapter Interface

Built-in Adapters

Static Adapter

Supabase Adapter

API Adapter

Adapter Factory

Creating Custom Adapters

Step 1: Implement the Interface

Step 2: Define Config Type

Step 3: Register in Factory

Step 4: Use in Config

Error Handling

Caching Strategy

Testing Adapters

On this page