config.yml.example

# Environment variables can be used in any config value with ${VAR} or ${VAR:-default} syntax:
#
#   log:
#     level: ${LOG_LEVEL:-info}
#     format: ${LOG_FORMAT:-text}
#
#   storage:
#     type: "s3"
#     s3:
#       endpoint: "${S3_ENDPOINT:-http://localhost:9000}"
#       bucket: "${S3_BUCKET?S3 bucket name must be set}"
#       access_key_id: "${AWS_ACCESS_KEY_ID?AWS access key ID must be set}"
#       secret_access_key: "${AWS_SECRET_ACCESS_KEY?AWS secret access key must be set}"
#
#   dump:
#     pg_dump_options:
#       dbname: "host=${DB_HOST:-localhost} user=${DB_USER:-postgres} dbname=${DB_NAME}"
#
# Supported syntax (POSIX Parameter Expansion):
#   ${VAR}          - value of VAR; empty string if unset
#   ${VAR:-default} - value of VAR, or "default" if unset or empty
#   ${VAR?message}  - value of VAR; exit with "message" if unset (use for required vars)
#   $$VAR           - literal $VAR (escape sequence)

common:
  pg_bin_path: "/usr/bin/"
  tmp_dir: "/tmp"

log:
  level: "info"
  format: "text"

storage:
  directory:
        path: "/tmp/pg_dump_test"
#  s3:
#    endpoint: "http://localhost:9000"
#    bucket: "testbucket"
#    region: "us-east-1"
#    access_key_id: "Q3AM3UQ867SPQQA43P2F"
#    secret_access_key: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"

dump:
  pg_dump_options:
    dbname: "host=/run/postgresql user=postgres dbname=demo"
    jobs: 10
    exclude-schema: "(\"teSt\"*|test*)"
    table: "bookings.flights"
    load-via-partition-root: true

  transformation:
    - schema: "public"
      name: "account"
      transformers:
        - name: "RandomInt"
          params:
            column: "id"
            engine: hash
            min: 1
            max: 2147483647

        - name: "RandomChoice"
          params:
            column: "gender"
            values:
              - "M"
              - "F"

        - name: "RandomPerson"
          params:
            columns:
              - name: "first_name"
                template: "{{ .FirstName }}"
              - name: "last_name"
                template: "{{ .LastName }}"
          dynamic_params:
            gender:
              column: gender

        - name: "Email"
          params:
            column: "email"
            engine: "hash"
            keep_original_domain: true
            keep_null: false
            local_part_template: "{{ first_name | lower }}.{{ last_name | lower }}"

        - name: "RandomDate"
          params:
            column: "birth_date"
            min: '{{ now | tsModify "-30 years" | .EncodeValue }}' # 1994
            max: '{{ now | tsModify "-18 years" | .EncodeValue }}' # 2006

        - name: "RandomDate"
          params:
            column: "created_at"
            max: "{{ now | .EncodeValue }}"
            truncate: "day"
          dynamic_params:
            min:
              column: "birth_date"
              template: '{{ .GetValue | tsModify "18 years" | .EncodeValue }}'

    - schema: "public"
      name: "orders"
      transformers:

        - name: "RandomInt"
          params:
            column: "account_id"
            engine: hash
            min: 1
            max: 2147483647

        - name: "NoiseNumeric"
          params:
              column: "total_price"
              decimal: 2
              min_ratio: 0.1
              max_ratio: 0.9

        - name: "NoiseDate"
          params:
            column: "created_at"
            max_ratio: "6 day"
            min_ratio: "1 day"
            truncate: "day"

        - name: "RandomDate"
          params:
            column: "paid_at"
            max: '{{ now | .EncodeValue }}'
            truncate: "day"
          dynamic_params:
            min:
              column: "created_at"

restore:
  pg_restore_options:
    jobs: 10
    dbname: "host=/run/postgresql user=postgres dbname=demo_restore"

  scripts:
    pre-data:
      - name: "pre-flight test script [1]"
        when: "before"
        query: "create table script_test(stage text)"
      - name: "pre-data before test script [1]"
        when: "before"
        query: "insert into script_test values('pre-data before')"
      - name: "pre-data after test script [1]"
        when: "after"
        query: "insert into script_test values('pre-data after')"
    data:
      - name: "data before test script [1]"
        when: "before"
        query: "insert into script_test values('data before')"
      - name: "data after test script [1]"
        when: "after"
        query: "insert into script_test values('data after')"
    post-data:
      - name: "post-data before test script [1]"
        when: "before"
        query: "insert into script_test values('post-data before')"
      - name: "post-data after test script [1]"
        when: "after"
        query: "insert into script_test values('post-data after')"