docker-compose.yml

### Port conventions (though there are many, many exceptions)
# 81xx - grapl services (and some `wait-for-it` http servers)
# 82xx - TBD
# 83xx - grapl plugin services, like grapl-aws-plugins
# 84xx - debugger ports (see vsc_debugger.py)

version: "3.8"
volumes:
  dgraph_export:

services:

  #
  # Infrastructure dependencies
  #

  grapl-master-graph-db:
    tty: false
    image: dgraph/standalone:v20.07.1
    ports:
      - "5080:5080"  # zero - used for `dgraph live` importing-data command in `integration_tests.rst`
      - "8000:8000"  # ratel
      - "8080:8080"  # alpha http
      - "9080:9080"  # alpha grpc
    logging:
      driver: none
    volumes:
      - type: volume
        source: dgraph_export
        # Hitting :8080/admin/export will force an export to be written to this directory.
        target: /dgraph/export

  # dev uses 1 big redis instance, prod has 1:1 redis per grapl service
  dev-shared-redis:  # maybe transitory, this will eventually match prod
    image: redis:latest
    command: |
      sh -c "
        # hack from https://stackoverflow.com/questions/54533308/disable-redis-persistence-in-docker
        # to disable persistence
        rm -f /data/dump.rdb && redis-server
      "
    ports:
      - "6379:6379" # redis default
    logging:
      driver: none

  sqs:
    image: localstack/localstack:0.11.5
    environment:
      - SERVICES=sqs:9324
      - HOSTNAME_EXTERNAL=sqs.us-east-1.amazonaws.com
      - IMAGE_NAME=localstack/localstack:0.11.5
    ports:
      - "9324:9324"
    networks:
      default:
        aliases:
          - sqs.us-east-1.amazonaws.com
    logging:
      driver: none

  secretsmanager:
    image: localstack/localstack:0.11.5
    environment:
      - PORT_WEB_UI=8099
      - SERVICES=secretsmanager:4584
      - HOSTNAME_EXTERNAL=secretsmanager.us-east-1.amazonaws.com
      - IMAGE_NAME=localstack/localstack:0.11.5

    ports:
      - "4584:4584"
      - "8099:8099"
    # logging:
    #   driver: none

  s3:
    image: minio/minio
    command: server /data
    ports:
      - "9000:9000"
    environment:
      - MINIO_ACCESS_KEY=minioadmin
      - MINIO_SECRET_KEY=minioadmin
      # Explicitly _dont_ specify MINIO_REGION_NAME - it breaks things for some reason.
      # Default is us-east-1 anyway
    networks:
      default:
        aliases:
          - minio
    logging:
      driver: none

  dynamodb:
    image: amazon/dynamodb-local
    logging:
      driver: none

  #
  # Rust services
  #

  grapl-metric-forwarder:
    image: grapl/grapl-metric-forwarder:${TAG:-latest}
    build:
      context: src/rust
      target: metric-forwarder-deploy
      args:
        - PROFILE=${PROFILE:-debug}
    tty: false
    environment:
      - "IS_LOCAL=True"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - RUST_LOG=${RUST_LOG:-ERROR}
    extra_hosts:
      - amazonaws:127.0.0.1

  grapl-sysmon-subgraph-generator:
    image: grapl/grapl-sysmon-subgraph-generator:${TAG:-latest}
    build:
      context: src/rust
      target: sysmon-subgraph-generator-deploy
      args:
        - PROFILE=${PROFILE:-debug}
    tty: false
    environment:
      - "BUCKET_PREFIX=local-grapl"
      - "IS_LOCAL=True"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - "SOURCE_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-sysmon-graph-generator-queue"
      - RUST_LOG=${RUST_LOG:-ERROR}
    links:
      - s3:minio
      - sqs:sqs.us-east-1.amazonaws.com
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1
    depends_on:
      - s3
      - sqs
      - grapl-provision

  grapl-osquery-subgraph-generator:
    image: grapl/grapl-osquery-subgraph-generator:${TAG:-latest}
    build:
      context: src/rust
      target: osquery-subgraph-generator-deploy
      args:
        - PROFILE=${PROFILE:-debug}
    tty: false
    environment:
      - "BUCKET_PREFIX=local-grapl"
      - "IS_LOCAL=True"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - "SOURCE_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-osquery-graph-generator-queue"
      - RUST_LOG=${RUST_LOG:-ERROR}
    links:
      - s3:minio
      - sqs:sqs.us-east-1.amazonaws.com
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1
    depends_on:
      - s3
      - sqs
      - grapl-provision

  grapl-node-identifier:
    image: grapl/grapl-node-identifier:${TAG:-latest}
    build:
      context: src/rust
      target: node-identifier-deploy
      args:
        - PROFILE=${PROFILE:-debug}
    environment:
      - "BUCKET_PREFIX=local-grapl"
      - "IS_LOCAL=True"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - RUST_LOG=${RUST_LOG:-ERROR}
      - "SOURCE_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-node-identifier-queue"
      - "GRAPH_MERGER_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-graph-merger-queue"
      - STATIC_MAPPING_TABLE=local-grapl-static_mapping_table
      - DYNAMIC_SESSION_TABLE=local-grapl-dynamic_session_table
      - PROCESS_HISTORY_TABLE=local-grapl-process_history_table
      - FILE_HISTORY_TABLE=local-grapl-file_history_table
      - INBOUND_CONNECTION_HISTORY_TABLE=local-grapl-inbound_connection_history_table
      - OUTBOUND_CONNECTION_HISTORY_TABLE=local-grapl-outbound_connection_history_table
      - NETWORK_CONNECTION_HISTORY_TABLE=local-grapl-network_connection_history_table
      - IP_CONNECTION_HISTORY_TABLE=local-grapl-ip_connection_history_table
      - ASSET_ID_MAPPINGS=local-grapl-asset_id_mappings
    tty: false
    links:
      - s3:minio
      - dynamodb:dynamodb
      - sqs:sqs.us-east-1.amazonaws.com
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1
    depends_on:
      - s3
      - dynamodb
      - sqs
      - grapl-provision

  grapl-node-identifier-retry-handler:
    image: grapl/grapl-node-identifier-retry-handler:${TAG:-latest}
    build:
      context: src/rust
      target: node-identifier-retry-handler-deploy
      args:
        - PROFILE=${PROFILE:-debug}
    environment:
      - "BUCKET_PREFIX=local-grapl"
      - "IS_LOCAL=True"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - RUST_LOG=${RUST_LOG:-ERROR}
      - "SOURCE_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-node-identifier-retry-queue"
      - "GRAPH_MERGER_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-graph-merger-queue"
      - STATIC_MAPPING_TABLE=local-grapl-static_mapping_table
      - DYNAMIC_SESSION_TABLE=local-grapl-dynamic_session_table
      - PROCESS_HISTORY_TABLE=local-grapl-process_history_table
      - FILE_HISTORY_TABLE=local-grapl-file_history_table
      - INBOUND_CONNECTION_HISTORY_TABLE=local-grapl-inbound_connection_history_table
      - OUTBOUND_CONNECTION_HISTORY_TABLE=local-grapl-outbound_connection_history_table
      - NETWORK_CONNECTION_HISTORY_TABLE=local-grapl-network_connection_history_table
      - IP_CONNECTION_HISTORY_TABLE=local-grapl-ip_connection_history_table
      - ASSET_ID_MAPPINGS=local-grapl-asset_id_mappings
    tty: false
    links:
      - s3:minio
      - dynamodb:dynamodb
      - sqs:sqs.us-east-1.amazonaws.com
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1
    depends_on:
      - s3
      - dynamodb
      - sqs
      - grapl-provision

  grapl-graph-merger:
    image: grapl/grapl-graph-merger:${TAG:-latest}
    build:
      context: src/rust
      target: graph-merger-deploy
      args:
        - PROFILE=${PROFILE:-debug}
    environment:
      - "BUCKET_PREFIX=local-grapl"
      - "IS_LOCAL=True"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - "SOURCE_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-graph-merger-queue"
      - "ANALYZER_DISPATCHER_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-analyzer-dispatcher-queue"
      - RUST_LOG=${RUST_LOG:-ERROR}
      - MG_ALPHAS=http://master_graph:9080
      - "GRAPL_SCHEMA_TABLE=local-grapl-grapl_schema_table"
    tty: false
    links:
      - s3:minio
      - sqs:sqs.us-east-1.amazonaws.com
      - grapl-master-graph-db:master_graph
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1
    depends_on:
      - s3
      - sqs
      - grapl-provision
      - grapl-master-graph-db

  grapl-analyzer-dispatcher:
    image: grapl/grapl-analyzer-dispatcher:${TAG:-latest}
    build:
      context: src/rust
      target: analyzer-dispatcher-deploy
      args:
        - PROFILE=${PROFILE:-debug}
    environment:
      - "BUCKET_PREFIX=local-grapl"
      - "IS_LOCAL=True"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - "SOURCE_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-analyzer-dispatcher-queue"
      - "ANALYZER_EXECUTOR_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-analyzer-executor-queue"
      - RUST_LOG=${RUST_LOG:-ERROR}
    tty: false
    links:
      - s3:minio
      - sqs:sqs.us-east-1.amazonaws.com
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1
    depends_on:
      - s3
      - sqs

  #
  # Python services
  #

  # Removed per https://grapl-internal.slack.com/archives/C017PLQ8TCZ/p1601574470186700
  # Add it back once Analyzer Deployer is ready for prime time!
  # analyzer-deployer:
  #   image: grapl/analyzer-deployer:${TAG:-latest}
  #   command: /bin/sh -c '. venv/bin/activate && cd /home/grapl/app && chalice local --no-autoreload --host=0.0.0.0 --port=8125'
  #   environment:
  #     - "IS_LOCAL=True"
  #     - GRPC_ENABLE_FORK_SUPPORT=1
  #     - "BUCKET_PREFIX=local-grapl"
  #     - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
  #   links:
  #     - s3:minio
  #     - sqs:sqs.us-east-1.amazonaws.com
  #   tty: true
  #   extra_hosts:
  #     - sqs:127.0.0.1
  #     - amazonaws:127.0.0.1
  #     - minio:127.0.0.1

  grapl-analyzer-executor:
    image: grapl/grapl-analyzer-executor:${TAG:-latest}
    build:
      context: src
      dockerfile: ./python/Dockerfile
      target: analyzer-executor-deploy
    command: /bin/sh -c '. venv/bin/activate && python3 analyzer_executor/src/run_local.py'
    environment:
      - DEBUG_SERVICES=${DEBUG_SERVICES}
      - "HITCACHE_ADDR=dev-shared-redis"
      - "HITCACHE_PORT=6379"
      - "MESSAGECACHE_ADDR=dev-shared-redis"
      - "MESSAGECACHE_PORT=6379"
      - "IS_LOCAL=True"
      - GRPC_ENABLE_FORK_SUPPORT=1
      - MG_ALPHAS=master_graph:9080
      - "IS_RETRY=False"
      - "BUCKET_PREFIX=local-grapl"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - "SOURCE_QUEUE_URL=http://sqs.us-east-1.amazonaws.com:9324/queue/grapl-analyzer-executor-queue"
      - AWS_REGION=us-east-1
      - "SQS_ENDPOINT=http://sqs.us-east-1.amazonaws.com:9324"
      - "SQS_ACCESS_KEY_ID=dummy_cred_aws_access_key_id"
      - "SQS_ACCESS_KEY_SECRET=dummy_cred_aws_secret_access_key"
      - "S3_ENDPOINT=http://s3:9000"
      - "S3_ACCESS_KEY_ID=minioadmin"
      - "S3_ACCESS_KEY_SECRET=minioadmin"
    links:
      - s3:minio
      - sqs:sqs.us-east-1.amazonaws.com
      - grapl-master-graph-db:master_graph
    tty: true
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1
    ports:
      - 8401:8401  # VSC Debugger

  grapl-engagement-creator:
    image: grapl/grapl-engagement-creator:${TAG:-latest}
    build:
      context: src
      dockerfile: ./python/Dockerfile
      target: engagement-creator-deploy
    command: /bin/sh -c '. venv/bin/activate && python3 engagement-creator/src/engagement-creator.py'
    environment:
      - "IS_LOCAL=True"
      - "BUCKET_PREFIX=local-grapl"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - MG_ALPHAS=master_graph:9080
    tty: true
    links:
      - s3:minio
      - sqs:sqs.us-east-1.amazonaws.com
      - grapl-master-graph-db:master_graph
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1

  grapl-engagement-edge:
    image: grapl/grapl-engagement-edge:${TAG:-latest}
    build:
      context: src
      dockerfile: ./python/Dockerfile
      target: engagement-edge-deploy
    command: |
      /bin/sh -c '
        wait-for-it grapl-provision:8126 --timeout=120 &&
        . venv/bin/activate &&
        cd /home/grapl/app &&
        chalice local --no-autoreload --host=0.0.0.0 --port=8900
      '
    ports:
      - "8900:8900"
    environment:
      - "IS_LOCAL=True"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - "UX_BUCKET_URL=localhost:3000"
      - "USER_AUTH_TABLE=local-grapl-user_auth_table"
      - "BUCKET_PREFIX=local-grapl"
    depends_on:
      - grapl-provision
    tty: true
    links:
      - grapl-master-graph-db:master_graph
      - secretsmanager:secretsmanager.us-east-1.amazonaws.com

  grapl-model-plugin-deployer:
    image: grapl/grapl-model-plugin-deployer:${TAG:-latest}
    build:
      context: src
      dockerfile: ./python/Dockerfile
      target: model-plugin-deployer-deploy
    command: /bin/sh -c '. venv/bin/activate && cd /home/grapl/app && chalice local --no-autoreload --host=0.0.0.0 --port=8123'
    ports:
      - "8123:8123"
    environment:
      - "IS_LOCAL=True"
      - "MG_ALPHAS=master_graph:9080"
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - "UX_BUCKET_URL=localhost"
    tty: true
    links:
      - s3:minio
      - sqs:sqs.us-east-1.amazonaws.com
      - secretsmanager:secretsmanager.us-east-1.amazonaws.com
      - grapl-master-graph-db:master_graph
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1

  grapl-dgraph-ttl:
    image: grapl/grapl-dgraph-ttl:${TAG:-latest}
    build:
      context: src
      dockerfile: ./python/Dockerfile
      target: dgraph-ttl-deploy
    command: /bin/sh -c '. venv/bin/activate && cd /home/grapl/app && chalice local --no-autoreload --host=0.0.0.0 --port=8124'
    ports:
      - "8124:8124"
    environment:
      - "IS_LOCAL=True"
      - MG_ALPHAS=master_graph:9080
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - GRAPL_DGRAPH_TTL_S=${GRAPL_DGRAPH_TTL_S:-31536000}
      - GRAPL_TTL_DELETE_BATCH_SIZE=${GRAPL_TTL_DELETE_BATCH_SIZE:-10}
    tty: true
    links:
      - grapl-master-graph-db:master_graph
    extra_hosts:
      - amazonaws:127.0.0.1

  #
  # JS/web services
  #

  grapl-engagement-view:
    image: syntaqx/serve
    volumes:
      - ./src/js/grapl-cdk/edge_ux_pre_replace:/var/www:ro
    command: "serve -dir /var/www"
    environment:
      - PORT=1234
    ports:
      - 1234:1234

  grapl-graphql-endpoint:
    image: grapl/grapl-graphql-endpoint:${TAG:-latest}
    build: 
      context: src/js/graphql_endpoint
      dockerfile: Dockerfile
      target: graphql-endpoint-deploy
    command: yarn start server
    environment:
      - MG_ALPHAS=master_graph:9080
      - IS_LOCAL=True
      - PORT=5000
      - JWT_SECRET_ID=JWT_SECRET_ID
    links:
      - grapl-master-graph-db:master_graph
      - secretsmanager:secretsmanager.us-east-1.amazonaws.com
    ports:
      - 5000:5000

  grapl-notebook:
    image: grapl/grapl-notebook:${TAG:-latest}
    build: 
      context: src
      dockerfile: ./python/Dockerfile
      target: grapl-notebook
    user: grapl
    environment:
      - MG_ALPHAS=master_graph:9080
    links:
      - grapl-master-graph-db:master_graph
    ports:
      - 8888:8888

  #
  # Utility services
  #

  grapl-provision:
    image: grapl/grapl-provision:${TAG:-latest}
    build: 
      context: src
      dockerfile: ./python/Dockerfile
      target: grapl-provision
    command: |
      /bin/bash -c "
        # it doesn't seem to like waiting for sqs
        # wait-for-it sqs:9324 &&
        wait-for-it secretsmanager:4584 &&
        wait-for-it s3:9000 &&
        wait-for-it dynamodb:8000 &&
        wait-for-it grapl-master-graph-db:8080 &&
        wait-for-it grapl-engagement-view:1234 &&
        . venv/bin/activate && 
        python /home/grapl/grapl_local_provision/provision_local_identity_table.py &&
        python /home/grapl/grapl_local_provision/grapl_provision.py &&
        # Host a server on this port, so that tests can wait-for-it
        python -m http.server 8126
      "
    environment:
      - GRAPL_LOG_LEVEL=${GRAPL_LOG_LEVEL:-ERROR}
      - MG_ALPHAS=master_graph:9080
    tty: true
    ports:
      - 8126:8126
    links:
      - grapl-master-graph-db:master_graph
      - s3:minio
      - sqs:sqs.us-east-1.amazonaws.com
      - secretsmanager:secretsmanager.us-east-1.amazonaws.com
      - dynamodb:dynamodb
    extra_hosts:
      - sqs:127.0.0.1
      - amazonaws:127.0.0.1
      - minio:127.0.0.1
    depends_on:
      - grapl-master-graph-db
      - s3
      - sqs
      - dynamodb
      - grapl-engagement-view

networks:
  default:
    name: grapl-network