CoreLog Ee

LogShipper ​Log​Shipper

yaml
type: "io.kestra.plugin.ee.core.log.logshipper"
yaml
id: logShipper
namespace: system

tasks:
  - id: shipLogs
    type: io.kestra.plugin.ee.core.log.LogShipper
    logLevelFilter: INFO
    lookbackPeriod: P1D
    offsetKey: logShipperOffset
    delete: false
    logExporters:
      - id: file
        type: io.kestra.plugin.ee.core.log.FileLogExporter

      - id: awsCloudWatch
        type: io.kestra.plugin.ee.aws.cloudwatch.LogExporter
        accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
        secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"
        region: us-east-1
        logGroupName: kestra
        logStreamName: production

      - id: S3LogExporter
        type: io.kestra.plugin.ee.aws.s3.LogExporter
        accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
        secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"
        region: "{{ vars.region }}"
        format: JSON
        bucket: logbucket
        logFilePrefix: kestra-log-file
        maxLinesPerFile: 1000000

      - id: googleOperationalSuite
        type: io.kestra.plugin.ee.gcp.operationalsuite.LogExporter
        projectId: my-gcp-project

      - id: gcs
        type: io.kestra.plugin.ee.gcp.gcs.LogExporter
        projectId: myProjectId
        format: JSON
        maxLinesPerFile: 10000
        bucket: my-bucket
        logFilePrefix: kestra-log-file
        chunk: 1000

      - id: azureMonitor
        type: io.kestra.plugin.ee.azure.monitor.LogExporter
        endpoint: https://endpoint-host.ingest.monitor.azure.com
        tenantId: "{{ secret('AZURE_TENANT_ID') }}"
        clientId: "{{ secret('AZURE_CLIENT_ID') }}"
        clientSecret: "{{ secret('AZURE_CLIENT_SECRET') }}"
        ruleId: dcr-69f0b123041d4d6e9f2bf72aad0b62cf
        streamName: kestraLogs

      - id: azureBlobStorage
        type: io.kestra.plugin.ee.azure.storage.LogExporter
        endpoint: https://myblob.blob.core.windows.net/
        tenantId: "{{ secret('AZURE_TENANT_ID') }}"
        clientId: "{{ secret('AZURE_CLIENT_ID') }}"
        clientSecret: "{{ secret('AZURE_CLIENT_SECRET') }}"
        containerName: logs
        format: JSON
        logFilePrefix: kestra-log-file
        maxLinesPerFile: 1000000
        chunk: 1000

      - id: datadog
        type: io.kestra.plugin.ee.datadog.LogExporter
        basePath: https://http-intake.logs.datadoghq.eu
        apiKey: "{{ secret('DATADOG_API_KEY') }}"

      - id: elasticsearch
        type: io.kestra.plugin.ee.elasticsearch.LogExporter
        indexName: kestra-logs
        connection:
          basicAuth:
            password: "{{ secret('ES_PASSWORD') }}"
            username: kestra_user
          hosts:
            - https://elastic.example.com:9200

      - id: opensearch
        type: io.kestra.plugin.ee.opensearch.LogExporter
        indexName: kestra-logs
        connection:
          basicAuth:
            password: "{{ secret('ES_PASSWORD') }}"
            username: kestra_user
          hosts:
            - https://elastic.example.com:9200

      - id: newRelic
        type: io.kestra.plugin.ee.newrelic.LogExporter
        basePath: https://log-api.newrelic.com
        apiKey: "{{ secret('NEWRELIC_API_KEY') }}"

      - id: openTelemetry
        type: io.kestra.plugin.ee.opentelemetry.LogExporter
        otlpEndpoint: http://otel-collector:4318/v1/logs
        authorizationHeaderName: Authorization
        authorizationHeaderValue: "Bearer {{ secret('OTEL_TOKEN') }}"

triggers:
  - id: dailySchedule
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "0 0 * * *"
    disabled: true
Properties
Min items 1
Default INFO
Default P1D
Format duration
Format date-time
SubType
Format duration
Default PT5M
Format duration
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Default DIRECT
Possible Values
DIRECTHTTPSOCKS
SubType string
SubType string
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Default JSON
Possible Values
IONJSON
Default kestra-log-file
Default 100000
Default PT15M
Format duration
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Default false
SubType integer
Default UTF-8
Default true
SubType string
Possible Values
REQUEST_HEADERSREQUEST_BODYRESPONSE_HEADERSRESPONSE_BODY
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Default PT15M
Format duration
SubType string
Min items 1
SubType string
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Default Kestra
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Default JSON
Possible Values
IONJSON
Default kestra-log-file
Default 100000
SubType string
Default ["https://www.googleapis.com/auth/cloud-platform"]
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
SubType string
Default ["https://www.googleapis.com/auth/cloud-platform"]
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default ION
Possible Values
IONJSON
Default kestra-log-file
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Default LogExporter
Default Kestra
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000
Default JSON
Possible Values
IONJSON
Default kestra-log-file
Default 100000
Validation RegExp ^[a-zA-Z0-9][a-zA-Z0-9_-]*
Min length 1
Default 1000