httpPolling

name: http-polling-basic-to-http-batch
version: 1.0.0
description: Poll a JSON array from an HTTP API and send it as one batch request.
tags:
  - http-polling
  - http-output
input:
  type: httpPolling
  schedule: "*/15 * * * *"
  endpoint: https://source.example.com/api/orders
  headers:
    Accept: application/json
  dataField: orders
filters: []
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/orders/import
  method: POST
  requestMode: batch
  headers:
    Content-Type: application/json
  success:
    statusCodes:
      - 200
      - 201
      - 202

name: http-polling-page-pagination
version: 1.0.0
description: Poll page-number pagination and normalize fields before output.
tags:
  - http-polling
  - pagination
input:
  type: httpPolling
  schedule: "0 */1 * * *"
  endpoint: https://source.example.com/api/customers
  dataField: data
  pagination:
    type: page
    param: page
    limitParam: per_page
    limit: 100
    totalPagesField: total_pages
filters:
  - type: mapping
    mappings:
      - source: id
        target: customer.id
        transforms:
          - op: toString
      - source: email
        target: customer.email
        transforms:
          - op: trim
          - op: lowercase
      - source: name
        target: customer.name
        onMissing: useDefault
        defaultValue: Unknown
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/customers
  method: POST
  requestMode: batch

name: http-polling-offset-pagination-state
version: 1.0.0
description: Poll offset pagination with timestamp and ID state persistence.
tags:
  - http-polling
  - state
input:
  type: httpPolling
  schedule: "*/10 * * * *"
  endpoint: https://source.example.com/api/events
  dataField: events
  pagination:
    type: offset
    param: offset
    limitParam: limit
    limit: 250
    totalField: total
  statePersistence:
    timestamp:
      enabled: true
      queryParam: updated_after
    id:
      enabled: true
      field: event.id
      queryParam: after_id
    storagePath: ./.cannectors-state
filters:
  - type: set
    target: metadata.source
    value: events-api
  - type: remove
    target:
      - debug
      - internal.notes
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/events
  method: POST
  requestMode: batch

name: http-polling-cursor-oauth2
version: 1.0.0
description: Poll cursor pagination with OAuth2 client credentials.
tags:
  - http-polling
  - oauth2
input:
  type: httpPolling
  schedule: "0 */6 * * *"
  endpoint: https://source.example.com/api/invoices
  dataField: items
  authentication:
    type: oauth2
    credentials:
      tokenUrl: https://source.example.com/oauth/token
      clientId: ${SOURCE_CLIENT_ID}
      clientSecret: ${SOURCE_CLIENT_SECRET}
      scope: invoices.read customers.read
  pagination:
    type: cursor
    param: cursor
    limitParam: limit
    limit: 100
    nextCursorField: next_cursor
filters:
  - type: condition
    expression: status == "paid"
    else:
      - type: drop
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/invoices
  method: POST
  requestMode: batch

name: http-polling-state-odata-filter
version: 1.0.0
description: Incremental polling through an OData filter, with the watermark rendered into the endpoint rather than a query parameter.
tags:
  - http-polling
  - state-persistence
  - templating
input:
  type: httpPolling
  schedule: "*/10 * * * *"
  # The persisted watermark reaches any part of the request, not only the query
  # parameter statePersistence names:
  #
  #   state.lastTimestamp   RFC3339, the epoch before anything is persisted
  #   state.lastId          absent until an ID has been persisted
  #
  # Spaces belong to the OData grammar — the runtime percent-encodes them.
  endpoint: https://source.example.com/api/orders?$filter=LastModifiedDate gt {{ state.lastTimestamp }}
  method: GET
  dataField: value
  headers:
    # Available in headers too. A custom header rather than If-Modified-Since:
    # RFC 9110 requires an HTTP-date there ("Sat, 15 Mar 2026 08:30:00 GMT") and
    # mandates that a malformed value be ignored, so sending the RFC3339
    # watermark would silently do nothing.
    X-Since: "{{ state.lastTimestamp }}"
  queryParams:
    # And in queryParams values.
    $orderby: LastModifiedDate asc
    $top: "200"
  statePersistence:
    timestamp:
      enabled: true
    id:
      enabled: true
      field: Id
    storagePath: ./.cannectors-state
filters: []
output:
  type: httpRequest
  endpoint: https://destination.example.com/api/orders/import
  method: POST
  requestMode: batch