http_call

name: http-call-error-marker-routing
version: 1.0.0
description: Enrich records, mark failures instead of losing them, and route rejects on the marker.
tags:
  - http-call
  - error-handling
  - condition
input:
  type: httpPolling
  schedule: "*/15 * * * *"
  endpoint: https://source.example.com/api/orders
  dataField: orders
filters:
  # onError: log lets the record continue and annotates it under the reserved
  # `_errors` list, instead of dropping it without trace (skip) or stopping the
  # whole batch for one bad record (fail).
  - type: http_call
    endpoint: https://profiles.example.com/api/customers/{customerId}
    method: GET
    onError: log
    keys:
      - field: customer.id
        paramType: path
        paramName: customerId
    dataField: profile
    mergeStrategy: merge
    cache:
      enabled: true
      maxSize: 10000
      ttlSeconds: 900
      key: customer.id
    retry:
      maxAttempts: 3
      delayMs: 500
      backoffMultiplier: 2
    # This API answers 409 while a profile is being re-indexed — a transient
    # condition, even though 4xx codes are functional by default. Declaring it
    # technical marks the record retryable so the next cycle picks it up.
    # The category of a code already on the right side of the line is kept:
    # a 404 stays `not_found`, it does not become a flat `validation`.
    errorClassification:
      technical:
        - 409
      functional:
        - 404

  # A functional failure will never succeed on replay: tag it for the dead
  # letter path so the next cycle does not pick it up again.
  # `_errors != nil` is the guard to write — the key is absent from records that
  # never failed, and expr rejects len(nil).
  - type: condition
    expression: "_errors != nil and _errors[0].retryable == false"
    then:
      - type: set
        target: routing.status
        value: REJECTED
    else:
      - type: set
        target: routing.status
        value: OK

  # Technical failures stay replayable: drop them from this run so the next
  # cycle retries them, rather than shipping a half-enriched record downstream.
  # They still appear in the end-of-run "Error markers" report: markers are
  # counted where they are created, so routing a record out does not erase the
  # failure that made you route it.
  - type: condition
    expression: "_errors != nil and _errors[0].retryable == true"
    then:
      - type: drop

output:
  type: httpRequest
  endpoint: https://destination.example.com/api/orders/enriched
  method: POST
  requestMode: batch

name: http-call-get-merge-cache
version: 1.0.0
description: Enrich each record with an HTTP GET response and merge it recursively.
tags:
  - http-call
  - cache
input:
  type: httpPolling
  schedule: "*/20 * * * *"
  endpoint: https://source.example.com/api/orders
  dataField: orders
filters:
  - type: http_call
    endpoint: https://profiles.example.com/api/customers/{customerId}
    method: GET
    keys:
      - field: customer.id
        paramType: path
        paramName: customerId
    dataField: profile
    mergeStrategy: merge
    cache:
      enabled: true
      maxSize: 10000
      ttlSeconds: 900
      key: customer.id
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/orders/enriched
  method: POST
  requestMode: batch

name: http-call-query-header-append
version: 1.0.0
description: Use query and header keys for an HTTP enrichment call and append the response.
tags:
  - http-call
  - append
input:
  type: httpPolling
  schedule: "*/20 * * * *"
  endpoint: https://source.example.com/api/tickets
  dataField: tickets
filters:
  - type: http_call
    endpoint: https://support.example.com/api/sla
    method: GET
    headers:
      X-Tenant: "{{record.tenantId}}"
    keys:
      - field: priority
        paramType: query
        paramName: priority
      - field: tenantId
        paramType: header
        paramName: X-Tenant-Id
    mergeStrategy: append
    resultKey: sla
    cache:
      enabled: true
      maxSize: 1000
      ttlSeconds: 300
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/tickets
  method: POST
  requestMode: batch

name: http-call-post-template-replace
version: 1.0.0
description: Use POST enrichment with a body template and replace overlapping response fields.
tags:
  - http-call
  - body-template
input:
  type: httpPolling
  schedule: "0 * * * *"
  endpoint: https://source.example.com/api/addresses
  dataField: addresses
filters:
  - type: http_call
    endpoint: https://geo.example.com/api/normalize
    method: POST
    headers:
      X-Request-Source: cannectors
    bodyTemplateFile: examples/assets/templates/geocode_request.json
    dataField: result
    mergeStrategy: replace
    cache:
      enabled: true
      maxSize: 5000
      ttlSeconds: 86400
      key: address.hash
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/addresses/normalized
  method: POST
  requestMode: batch

name: call-filters-shared-merge-contract
version: 1.0.0
description: The three call filters share one merge contract - mergeStrategy append nests the whole response under resultKey.
tags:
  - http-call
  - soap-call
  - sql-call
  - append
input:
  type: httpPolling
  schedule: "*/15 * * * *"
  endpoint: https://source.example.com/api/orders
  dataField: orders
filters:
  - type: http_call
    endpoint: https://crm.example.com/api/customers
    method: GET
    keys:
      - field: customerId
        paramType: query
        paramName: id
    mergeStrategy: append
    resultKey: customer
    cache:
      enabled: true
      maxSize: 1000
      ttlSeconds: 300
      key: "customer:{{record.customerId}}"
  - type: soap_call
    endpoint: https://erp.example.com/contracts
    soapAction: urn:GetContract
    operation: GetContract
    body: |
      <m:GetContract xmlns:m="urn:contracts">
        <m:CustomerId>{{record.customerId}}</m:CustomerId>
      </m:GetContract>
    dataField: Envelope.Body.GetContractResponse.Contract
    mergeStrategy: append
    resultKey: contract
    cache:
      enabled: true
      maxSize: 1000
      ttlSeconds: 300
      key: "contract:{{record.customerId}}"
  - type: sql_call
    connectionStringRef: ${REFERENCE_DATABASE_URL}
    driver: postgres
    query: SELECT tier, discount FROM pricing WHERE customer_id = $1
    parameters:
      - record.customerId
    mergeStrategy: append
    resultKey: pricing
    cache:
      enabled: true
      maxSize: 1000
      ttlSeconds: 300
      key: "pricing:{{record.customerId}}"
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/orders/enriched
  method: POST
  requestMode: batch
  headers:
    Content-Type: application/json

name: http-call-datafield-list-loop
version: 1.0.0
description: Enrich each record with a list-shaped HTTP response, then iterate the list with the loop filter.
tags:
  - http-call
  - data-field
  - loop
input:
  type: httpPolling
  schedule: "*/10 * * * *"
  endpoint: https://source.example.com/api/orders
  dataField: orders
filters:
  - type: http_call
    endpoint: https://catalog.example.com/api/matches
    method: GET
    dataField: results
    keys:
      - field: reference
        paramType: query
        paramName: reference
    mergeStrategy: append
    resultKey: matches
    cache:
      enabled: true
      maxSize: 1000
      ttlSeconds: 300
      key: "matches:{{record.reference}}"
  - type: loop
    field: matches
    itemName: match
    filters:
      - type: mapping
        mappings:
          - source: match.productCode
            target: match.sku
      - type: remove
        target:
          - match.productCode
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/orders/matched
  method: POST
  requestMode: batch