From 0e95e19160ae9573ebe679308e09cc5f41d26f9b Mon Sep 17 00:00:00 2001 From: h3xit <85270067+h3xitsec@users.noreply.github.com> Date: Mon, 4 Nov 2024 09:41:02 -0500 Subject: [PATCH] Github Action + Final Swarm configuration (#1) * . * added workflow * fix trigger * add image build * add PAT usage * add workflow permissions * . * change ghcr repo name * change image name * add build script * shabang * remove secrets from build * test build * debug * debug * debug * debug 2 * fix cp * fix dockerfile path * add arm64 build * add worker build * fix * add gha cache * . * remove docker build * .. * refactor * add all image build * fix path * add missing images * fix image cmd * update gitignore * remove shell.nix * final swarm + ghcr image push --------- Co-authored-by: h3xit --- .github/workflows/build-push.yaml | 110 +++++++++++ .gitignore | 21 ++ Ansible/deploy_stack.yaml | 23 +++ Ansible/refresh_nodes.yaml | 7 - Ansible/remove_stack.yaml | 23 +++ Ansible/setup_nodes.yaml | 20 +- Ansible/start_nodes.yaml | 4 - Ansible/start_processor.yaml | 10 - Ansible/start_workers.yaml | 14 -- Ansible/stop_nodes.yaml | 3 - Ansible/stop_processor.yaml | 8 - Ansible/stop_workers.yaml | 8 - Ansible/update_h3xrecon.yaml | 19 -- build.sh | 146 +++++++------- build_processor.sh | 31 --- build_worker.sh | 23 --- .../docker-compose.yaml | 68 +------ shell.nix | 49 ----- src/BaseImage/Dockerfile | 8 +- src/DataProcessor/Dockerfile | 6 +- src/JobProcessor/Dockerfile | 6 +- src/Logger/Dockerfile | 6 +- src/Worker/Dockerfile | 46 ++--- src/docker-compose.processor.yaml | 179 ------------------ src/docker-compose.workers.yaml | 66 ------- src/nats/Dockerfile | 4 +- src/pgsql/Dockerfile | 2 +- src/pgsql/psql_dump.sql | 26 --- syncvps.sh | 5 - 29 files changed, 304 insertions(+), 637 deletions(-) create mode 100644 .github/workflows/build-push.yaml create mode 100644 .gitignore create mode 100644 Ansible/deploy_stack.yaml delete mode 100644 Ansible/refresh_nodes.yaml create mode 100644 Ansible/remove_stack.yaml delete mode 100644 Ansible/start_nodes.yaml delete mode 100644 Ansible/start_processor.yaml delete mode 100644 Ansible/start_workers.yaml delete mode 100644 Ansible/stop_nodes.yaml delete mode 100644 Ansible/stop_processor.yaml delete mode 100644 Ansible/stop_workers.yaml delete mode 100644 Ansible/update_h3xrecon.yaml delete mode 100755 build_processor.sh delete mode 100755 build_worker.sh rename src/docker-compose_swarm.yaml => deploy/docker-compose.yaml (60%) delete mode 100644 shell.nix delete mode 100644 src/docker-compose.processor.yaml delete mode 100644 src/docker-compose.workers.yaml delete mode 100755 syncvps.sh diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml new file mode 100644 index 0000000..95908a4 --- /dev/null +++ b/.github/workflows/build-push.yaml @@ -0,0 +1,110 @@ +name: Build and Push Docker Images + +on: + push: + branches: + - master + workflow_dispatch: +permissions: + packages: write + contents: read +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run build script + run: bash build.sh + + - name: Build and push base image + uses: docker/build-push-action@v4 + with: + context: ./build/BaseImage + file: ./build/BaseImage/Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_base:latest + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + - name: Build and push worker image + uses: docker/build-push-action@v4 + with: + context: ./build/Worker + file: ./build/Worker/Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_worker:latest + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + - name: Build and push data processor image + uses: docker/build-push-action@v4 + with: + context: ./build/DataProcessor + file: ./build/DataProcessor/Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_dataprocessor:latest + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + + - name: Build and push job processor image + uses: docker/build-push-action@v4 + with: + context: ./build/JobProcessor + file: ./build/JobProcessor/Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_jobprocessor:latest + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + + - name: Build and push logger image + uses: docker/build-push-action@v4 + with: + context: ./build/Logger + file: ./build/Logger/Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_logger:latest + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + + - name: Build and push nats image + uses: docker/build-push-action@v4 + with: + context: ./build/nats + file: ./build/nats/Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_nats:latest + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + + - name: Build and push pgsql image + uses: docker/build-push-action@v4 + with: + context: ./build/pgsql + file: ./build/pgsql/Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_pgsql:latest + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1fd1280 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +venv/ +notes.md +.env +.psqlpass +*.log +env.sh +.env +secrets/ +build/ +shell.nix +vault.yaml +tailscale_vault.yaml +old_keep/ +*.key +*.pub +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so diff --git a/Ansible/deploy_stack.yaml b/Ansible/deploy_stack.yaml new file mode 100644 index 0000000..88af182 --- /dev/null +++ b/Ansible/deploy_stack.yaml @@ -0,0 +1,23 @@ +- name: Start h3xrecon stack + hosts: processor + vars_files: + - vault.yaml + tasks: + - delegate_to: localhost + ansible.builtin.copy: + dest: /tmp/h3xrecon_processor.key + content: "{{ PROCESSOR_PRIVATE_KEY }}" + mode: 0600 + - name: Deploy Docker stack + delegate_to: localhost + community.docker.docker_stack: + state: present + name: h3xrecon + compose: + - ../deploy/docker-compose.yaml + environment: + DOCKER_HOST: ssh://{{ ansible_host }} + - delegate_to: localhost + ansible.builtin.file: + path: /tmp/h3xrecon_processor.key + state: absent \ No newline at end of file diff --git a/Ansible/refresh_nodes.yaml b/Ansible/refresh_nodes.yaml deleted file mode 100644 index 43dec8b..0000000 --- a/Ansible/refresh_nodes.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- hosts: processor - gather_facts: false - - tasks: - - ansible.builtin.include_tasks: tasks/test_all_nodes.yaml - - ansible.builtin.include_tasks: tasks/build_project.yaml - - ansible.builtin.include_tasks: tasks/sync_project.yaml diff --git a/Ansible/remove_stack.yaml b/Ansible/remove_stack.yaml new file mode 100644 index 0000000..b92c7d8 --- /dev/null +++ b/Ansible/remove_stack.yaml @@ -0,0 +1,23 @@ +- name: Remove h3xrecon stack + hosts: processor + vars_files: + - vault.yaml + tasks: + - delegate_to: localhost + ansible.builtin.copy: + dest: /tmp/h3xrecon_processor.key + content: "{{ PROCESSOR_PRIVATE_KEY }}" + mode: 0600 + - name: Deploy Docker stack + delegate_to: localhost + community.docker.docker_stack: + state: absent + name: h3xrecon + compose: + - ../deploy/docker-compose.yaml + environment: + DOCKER_HOST: ssh://{{ ansible_host }} + - delegate_to: localhost + ansible.builtin.file: + path: /tmp/h3xrecon_processor.key + state: absent \ No newline at end of file diff --git a/Ansible/setup_nodes.yaml b/Ansible/setup_nodes.yaml index 336e245..bf3aec3 100644 --- a/Ansible/setup_nodes.yaml +++ b/Ansible/setup_nodes.yaml @@ -48,11 +48,11 @@ path: "{{ h3xrecon_target_directory }}/" state: absent - - name: Sync codebase - ansible.builtin.synchronize: - src: "{{ h3xrecon_source_directory }}" - dest: "{{ h3xrecon_target_directory }}" - rsync_opts: '--exclude=.git --exclude="**/__pycache__/*" --exclude=venv' + # - name: Sync codebase + # ansible.builtin.synchronize: + # src: "{{ h3xrecon_source_directory }}" + # dest: "{{ h3xrecon_target_directory }}" + # rsync_opts: '--exclude=.git --exclude="**/__pycache__/*" --exclude=venv' - name: Setup Python Virtual Environment ansible.builtin.pip: @@ -67,9 +67,9 @@ groups: docker append: yes - - name: Enable arm64 image building - become: true - ansible.builtin.shell: docker run --privileged --rm tonistiigi/binfmt --install all + # - name: Enable arm64 image building + # become: true + # ansible.builtin.shell: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Build Images - ansible.builtin.include_tasks: tasks/build_container_images.yaml \ No newline at end of file + # - name: Build Images + # ansible.builtin.include_tasks: tasks/build_container_images.yaml \ No newline at end of file diff --git a/Ansible/start_nodes.yaml b/Ansible/start_nodes.yaml deleted file mode 100644 index f733d1b..0000000 --- a/Ansible/start_nodes.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- ansible.builtin.import_playbook: "build_project.yaml" -- ansible.builtin.import_playbook: "start_processor.yaml" -- ansible.builtin.import_playbook: "start_workers.yaml" \ No newline at end of file diff --git a/Ansible/start_processor.yaml b/Ansible/start_processor.yaml deleted file mode 100644 index 9548f6a..0000000 --- a/Ansible/start_processor.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Start h3xrecon processor - hosts: processor - tasks: - - ansible.builtin.include_tasks: tasks/build_project.yaml - - - name: Start Recon Processor - community.docker.docker_compose_v2: - project_src: "{{ h3xrecon_target_directory }}" - build: never - state: present \ No newline at end of file diff --git a/Ansible/start_workers.yaml b/Ansible/start_workers.yaml deleted file mode 100644 index 1f43832..0000000 --- a/Ansible/start_workers.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- name: Start h3xrecon workers - hosts: workers - tasks: - - name: Sync Project - ansible.builtin.synchronize: - src: "{{ h3xrecon_source_directory }}" - dest: "{{ h3xrecon_target_directory }}" - rsync_opts: '--exclude=.git --exclude="**/__pycache__/*" --exclude=venv' - - - name: Start Recon Workers - community.docker.docker_compose_v2: - project_src: "{{ h3xrecon_target_directory }}" - build: never - state: present \ No newline at end of file diff --git a/Ansible/stop_nodes.yaml b/Ansible/stop_nodes.yaml deleted file mode 100644 index e5cd8e3..0000000 --- a/Ansible/stop_nodes.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- ansible.builtin.import_playbook: "stop_processor.yaml" -- ansible.builtin.import_playbook: "stop_workers.yaml" \ No newline at end of file diff --git a/Ansible/stop_processor.yaml b/Ansible/stop_processor.yaml deleted file mode 100644 index 9cb4a5c..0000000 --- a/Ansible/stop_processor.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- name: Stop h3xrecon processor - hosts: processor - tasks: - - name: Start Recon Processor - community.docker.docker_compose_v2: - project_src: "{{ h3xrecon_target_directory }}" - build: never - state: absent \ No newline at end of file diff --git a/Ansible/stop_workers.yaml b/Ansible/stop_workers.yaml deleted file mode 100644 index 92d3231..0000000 --- a/Ansible/stop_workers.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- name: Stop h3xrecon workers - hosts: workers - tasks: - - name: Stop Recon Workers - community.docker.docker_compose_v2: - project_src: "{{ h3xrecon_target_directory }}" - build: never - state: absent \ No newline at end of file diff --git a/Ansible/update_h3xrecon.yaml b/Ansible/update_h3xrecon.yaml deleted file mode 100644 index a9cccfb..0000000 --- a/Ansible/update_h3xrecon.yaml +++ /dev/null @@ -1,19 +0,0 @@ -- name: Sync Project Packages - hosts: processor - gather_facts: false - max_fail_percentage: 0 - - vars_files: - - vault.yaml - - vars: - linux_apt_packages: - - docker.io - - "{{ h3xrecon_dockercompose_pkg }}" - - python3-venv - - tasks: - - ansible.builtin.include_tasks: tasks/test_all_nodes.yaml - - ansible.builtin.include_tasks: tasks/build_project.yaml - - ansible.builtin.include_tasks: tasks/sync_project.yaml - - ansible.builtin.include_tasks: tasks/build_container_images.yaml \ No newline at end of file diff --git a/build.sh b/build.sh index 704fd07..1b7a4a4 100755 --- a/build.sh +++ b/build.sh @@ -1,78 +1,84 @@ #!/etc/profiles/per-user/h3x/bin/bash +trap 'echo Exited!; exit;' SIGINT SIGTERM +unset DOCKER_HOST +echo "====================================" +echo " Building h3xrecon " +echo "====================================" -echo "Building h3xrecon" +echo "------------------------------------" +echo " Staging build directory " +echo "------------------------------------" -sudo rm -f build/requirements.txt +sudo rm -rf build/* +mkdir -p ./build -cp src/docker-compose_swarm.yaml build/docker-compose.yaml -cp src/requirements_node.txt build/requirements.txt -cp -r src/BaseImage build/BaseImage -cp -r secrets build/ -echo "H3XRECON_PROCESSOR_IP=localhost" > build/.env -cat .env >> build/.env - - -echo "Building Processor Package" - -sudo rm -rf build/JobProcesor -sudo rm -rf build/DataProcessor -sudo rm -rf build/Logger -sudo rm -rf build/Nats -sudo rm -rf build/Postgres - -mkdir build/JobProcesor -mkdir build/DataProcessor -mkdir build/Logger -mkdir build/Nats -mkdir build/Postgres - - -## Processor -# Job Processor -cp -r src/DataProcessor build/Processor/ -cp -r src/JobProcessor build/Processor/ -cp -r src/Logger build/Processor/ -cp -r src/pgsql build/Processor/ -cp -r src/nats build/Processor/ -cp -r src/DatabaseManager build/Processor/ -cp -r src/QueueManager build/Processor/ -cp -r src/BaseImage build/Processor/ -cp -r secrets build/Processor/ -cp -r src/tailscale build/Processor/ -cp src/docker-compose.processor.yaml build/Processor/docker-compose.yaml -cp src/requirements_processor.txt build/Processor/requirements.txt -echo "H3XRECON_PROCESSOR_IP=localhost" > build/Processor/.env -cat .env >> build/Processor/.env - -mv build/Processor/JobProcessor/Dockerfile build/Processor/Dockerfile.job_processor -mv build/Processor/DataProcessor/Dockerfile build/Processor/Dockerfile.data_processor -mv build/Processor/Logger/Dockerfile build/Processor/Dockerfile.logger - - -find ./build/Processor -type d -name __pycache__ | xargs -i{} sh -c "rm -rf {}" - -echo "Processor: Done" - -echo "Building Worker Package" - -sudo rm -rf build/Worker -mkdir build/Worker - -## Worker -cp -r src/Worker build/Worker/ -mv build/Worker/Worker/Dockerfile build/Worker/Dockerfile +cp -r src/BaseImage ./build/ +cp -r src/Worker ./build/ cp -r src/DatabaseManager build/Worker/ cp -r src/QueueManager build/Worker/ -cp -r src/BaseImage build/Worker/ -cp -r secrets build/Worker/ -cp -r src/tailscale build/Worker/ -cp src/docker-compose.workers.yaml build/Worker/docker-compose.yaml cp src/requirements_workers.txt build/Worker/requirements.txt -echo "H3XRECON_PROCESSOR_IP=processor" > build/Worker/.env -cat .env >> build/Worker/.env - -find ./build/Worker -type d -name __pycache__ | xargs -i{} sh -c "rm -rf {}" - -echo "Worker: Done" -echo "Build Completed" \ No newline at end of file +cp -r src/DataProcessor ./build/ +cp -r src/QueueManager build/DataProcessor/ +cp src/requirements_processor.txt build/DataProcessor/requirements.txt + +cp -r src/JobProcessor ./build/ +cp -r src/QueueManager build/JobProcessor/ +cp src/requirements_processor.txt build/JobProcessor/requirements.txt + +cp -r src/Logger ./build/ +cp -r src/DatabaseManager build/Logger/ +cp -r src/QueueManager build/Logger/ +cp src/requirements_processor.txt build/Logger/requirements.txt + +cp -r src/nats build/nats +cp -r src/pgsql build/pgsql + +echo "------------------------------------" +echo " Staging build directory completed " +echo "------------------------------------" + +if [ -z "$GITHUB_ACTIONS" ]; then + echo "====================================" + echo " Building docker images " + echo "====================================" + + echo "------------------------------------" + echo " Building BaseImage " + echo "------------------------------------" + docker buildx build --push --file ./build/BaseImage/Dockerfile --platform linux/amd64,linux/arm64 --tag ghcr.io/h3xitsec/h3xrecon_base:latest ./build/BaseImage + + echo "------------------------------------" + echo " Building Worker " + echo "------------------------------------" + docker buildx build --push --file ./build/Worker/Dockerfile --platform linux/amd64,linux/arm64 --tag ghcr.io/h3xitsec/h3xrecon_worker:latest ./build/Worker + + echo "------------------------------------" + echo " Building DataProcessor " + echo "------------------------------------" + docker buildx build --push --file ./build/DataProcessor/Dockerfile --platform linux/amd64 --tag ghcr.io/h3xitsec/h3xrecon_dataprocessor:latest ./build/DataProcessor + + echo "------------------------------------" + echo " Building JobProcessor " + echo "------------------------------------" + docker buildx build --push --file ./build/JobProcessor/Dockerfile --platform linux/amd64 --tag ghcr.io/h3xitsec/h3xrecon_jobprocessor:latest ./build/JobProcessor + + echo "------------------------------------" + echo " Building Logger " + echo "------------------------------------" + docker buildx build --push --file ./build/Logger/Dockerfile --platform linux/amd64 --tag ghcr.io/h3xitsec/h3xrecon_logger:latest ./build/Logger + + echo "------------------------------------" + echo " Building Nats " + echo "------------------------------------" + docker buildx build --push --file ./build/nats/Dockerfile --platform linux/amd64 --tag ghcr.io/h3xitsec/h3xrecon_nats:latest ./build/nats + + echo "------------------------------------" + echo " Building Pgsql " + echo "------------------------------------" + docker buildx build --push --file ./build/pgsql/Dockerfile --platform linux/amd64 --tag ghcr.io/h3xitsec/h3xrecon_pgsql:latest ./build/pgsql + + echo "====================================" + echo " Docker build commands completed! " + echo "====================================" +fi diff --git a/build_processor.sh b/build_processor.sh deleted file mode 100755 index 1c9c0cc..0000000 --- a/build_processor.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/etc/profiles/per-user/h3x/bin/bash - -echo "Building Processor Package" - -sudo rm -rf build/Processor -mkdir build/Processor - -## Processor -cp -r src/DataProcessor build/Processor/ -cp -r src/JobProcessor build/Processor/ -cp -r src/Logger build/Processor/ -cp -r src/pgsql build/Processor/ -cp -r src/nats build/Processor/ -cp -r src/DatabaseManager build/Processor/ -cp -r src/QueueManager build/Processor/ -cp -r src/BaseImage build/Processor/ -cp -r secrets build/Processor/ -cp -r src/tailscale build/Processor/ -cp src/docker-compose.processor.yaml build/Processor/docker-compose.yaml -cp src/requirements_processor.txt build/Processor/requirements.txt -echo "H3XRECON_PROCESSOR_IP=localhost" > build/Processor/.env -cat .env >> build/Processor/.env - -mv build/Processor/JobProcessor/Dockerfile build/Processor/Dockerfile.job_processor -mv build/Processor/DataProcessor/Dockerfile build/Processor/Dockerfile.data_processor -mv build/Processor/Logger/Dockerfile build/Processor/Dockerfile.logger - - -find ./build/Processor -type d -name __pycache__ | xargs -i{} sh -c "rm -rf {}" - -echo "Processor: Done" \ No newline at end of file diff --git a/build_worker.sh b/build_worker.sh deleted file mode 100755 index e07b8c5..0000000 --- a/build_worker.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/etc/profiles/per-user/h3x/bin/bash - -echo "Building Worker Package" - -sudo rm -rf build/Worker -mkdir build/Worker - -## Worker -cp -r src/Worker build/Worker/ -mv build/Worker/Worker/Dockerfile build/Worker/Dockerfile -cp -r src/DatabaseManager build/Worker/ -cp -r src/QueueManager build/Worker/ -cp -r src/BaseImage build/Worker/ -cp -r secrets build/Worker/ -cp -r src/tailscale build/Worker/ -cp src/docker-compose.workers.yaml build/Worker/docker-compose.yaml -cp src/requirements_workers.txt build/Worker/requirements.txt -echo "H3XRECON_PROCESSOR_IP=processor" > build/Worker/.env -cat .env >> build/Worker/.env - -find ./build/Worker -type d -name __pycache__ | xargs -i{} sh -c "rm -rf {}" - -echo "Worker: Done" \ No newline at end of file diff --git a/src/docker-compose_swarm.yaml b/deploy/docker-compose.yaml similarity index 60% rename from src/docker-compose_swarm.yaml rename to deploy/docker-compose.yaml index 2bcf0a0..ce33e0c 100644 --- a/src/docker-compose_swarm.yaml +++ b/deploy/docker-compose.yaml @@ -1,36 +1,4 @@ services: - # tailscale: - # image: tailscale/tailscale:latest - # environment: - # - TS_AUTH_KEY_FILE=/run/secrets/tailscale_auth_key - # - TS_AUTH_KEY= - # - TS_HOSTNAME=processor - # - TS_STATE_DIR=/var/lib/tailscale - # - TS_EXTRA_ARGS=--reset - # - TS_USERSPACE=false - # - TS_SERVE_CONFIG=./serve.yaml - # - TS_SERVE_PORT_RANGES=4222,5432,6379 - # volumes: - # - tailscale_state:/var/lib/tailscale - # - /dev/net/tun:/dev/net/tun - # - ./tailscale/serve.yaml:/var/lib/tailscale/serve.yaml - # - ./tailscale/entrypoint.sh:/entrypoint.sh - # cap_add: - # - net_admin - # - sys_module - # secrets: - # - tailscale_auth_key - # restart: unless-stopped - # entrypoint: ["/bin/sh", "/entrypoint.sh"] - # # logging: - # # driver: "none" - # privileged: true - # healthcheck: - # test: ["CMD", "tailscale", "status"] - # interval: 10s - # timeout: 5s - # retries: 3 - redis: image: redis:6.2-alpine volumes: @@ -56,12 +24,11 @@ services: - POSTGRES_PASSWORD_FILE=/run/secrets/postgresql_db_password - POSTGRES_USER=h3xrecon - POSTGRES_DB=h3xrecon - image: 127.0.0.1:5000/pgsql:latest + image: ghcr.io/h3xitsec/h3xrecon_pgsql:latest secrets: - postgresql_db_password volumes: - pgsql_data:/var/lib/postgresql/data - - ${PWD}/Processor/pgsql/psql_dump.sql:/docker-entrypoint-initdb.d/psql_dump.sql ports: - target: 5432 published: 5432 @@ -80,10 +47,9 @@ services: - node.labels.h3xrecon_role == processor nats: - image: 127.0.0.1:5000/nats:latest + image: ghcr.io/h3xitsec/h3xrecon_nats:latest volumes: - nats_data:/data - - ${PWD}/Processor/nats/init.sh:/init.sh ports: - target: 4222 published: 4222 @@ -101,16 +67,14 @@ services: logger: env_file: .env - image: 127.0.0.1:5000/logger:latest + image: ghcr.io/h3xitsec/h3xrecon_logger:latest secrets: - postgresql_db_password - volumes: - - ${PWD}/Processor/Logger:/app depends_on: - nats:service_healthy - pgsql:service_healthy environment: - - LOGURU_LEVEL=DEBUG + - LOGURU_LEVEL=INFO deploy: placement: constraints: @@ -118,18 +82,15 @@ services: job_processor: env_file: .env - image: 127.0.0.1:5000/job_processor:latest + image: ghcr.io/h3xitsec/h3xrecon_jobprocessor:latest secrets: - postgresql_db_password - volumes: - - ${PWD}/Processor/JobProcessor:/app/JobProcessor - - ${PWD}/Processor/QueueManager:/app/QueueManager depends_on: - tailscale:service_healthy - nats:service_healthy - redis:service_healthy environment: - - LOGURU_LEVEL=DEBUG + - LOGURU_LEVEL=INFO deploy: mode: replicated replicas: 1 @@ -139,18 +100,15 @@ services: data_processor: env_file: .env - image: 127.0.0.1:5000/data_processor:latest + image: ghcr.io/h3xitsec/h3xrecon_dataprocessor:latest secrets: - postgresql_db_password - volumes: - - ${PWD}/Processor/DataProcessor:/app/DataProcessor - - ${PWD}/Processor/QueueManager:/app/QueueManager depends_on: - tailscale:service_healthy - nats:service_healthy - redis:service_healthy environment: - - LOGURU_LEVEL=DEBUG + - LOGURU_LEVEL=INFO deploy: mode: replicated replicas: 1 @@ -160,15 +118,11 @@ services: worker: env_file: .env - image: 127.0.0.1:5000/worker:latest - volumes: - - /home/ubuntu/h3xrecon/Worker:/app/Worker - - /home/ubuntu/h3xrecon/Worker/QueueManager:/app/QueueManager - - /home/ubuntu/h3xrecon/Worker/DatabaseManager:/app/DatabaseManager + image: ghcr.io/h3xitsec/h3xrecon_worker:latest depends_on: - tailscale:service_healthy environment: - - LOGURU_LEVEL=DEBUG + - LOGURU_LEVEL=INFO secrets: - postgresql_db_password deploy: @@ -185,7 +139,5 @@ volumes: tailscale_state: {} secrets: - tailscale_auth_key: - file: ./secrets/tailscale_auth_key_processor.txt postgresql_db_password: file: ./secrets/postgresql_db_password.txt \ No newline at end of file diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 3ac178f..0000000 --- a/shell.nix +++ /dev/null @@ -1,49 +0,0 @@ -{ pkgs ? import {} }: - -pkgs.mkShellNoCC { - packages = with pkgs; [ - (python3.withPackages (ps: [ - ps.asyncpg - ps.certifi - ps.charset-normalizer - ps.idna - ps.loguru - ps.nats-py - ps.requests - ps.urllib3 - ps.redis - ])) - curl - nodejs_18 - jq - ipinfo - postgresql - nats-server - natscli - libz - go - gcc - ]; - shellHook = '' - export PGPASSWORD="h3xrecon" - export H3XRECON_PROCESSOR_IP=100.122.156.98 - export H3XRECON_REDIS_SERVER=$H3XRECON_PROCESSOR_IP - export H3XRECON_REDIS_PORT=6379 - export H3XRECON_NATS_SERVER=$H3XRECON_PROCESSOR_IP - export H3XRECON_NATS_PORT=4222 - export H3XRECON_DB_HOST=$H3XRECON_PROCESSOR_IP - export H3XRECON_DB_PORT=5432 - export H3XRECON_DB_USER=h3xrecon - export H3XRECON_DB_PASSWORD=h3xrecon - export H3XRECON_DB_NAME=h3xrecon - export LOGURU_LEVEL=DEBUG - export DOCKER_HOST=ssh://recon - export PATH="./Worker/bin:~/.local/share/go/bin:$PATH" - export LD_LIBRARY_PATH="/run/current-system/sw/share/nix-ld/lib" - source ./venv/bin/activate - python -m pip uninstall -y httpx > /dev/null 2>&1 - go install -v github.com/projectdiscovery/pdtm/cmd/pdtm@latest - pdtm -s -bp ~/.local/share/go/bin -i httpx > /dev/null 2>&1 - ''; -} - diff --git a/src/BaseImage/Dockerfile b/src/BaseImage/Dockerfile index 81777c1..ec8a761 100644 --- a/src/BaseImage/Dockerfile +++ b/src/BaseImage/Dockerfile @@ -12,4 +12,10 @@ COPY requirements.txt /app/requirements.txt ENV PYTHONPATH="/app" -RUN pip install --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --no-cache-dir -r requirements.txt \ No newline at end of file +RUN pip install \ + --upgrade \ + --trusted-host pypi.org \ + --trusted-host pypi.python.org \ + --trusted-host files.pythonhosted.org \ + --no-cache-dir \ + -r requirements.txt \ No newline at end of file diff --git a/src/DataProcessor/Dockerfile b/src/DataProcessor/Dockerfile index 2975660..b62ab52 100644 --- a/src/DataProcessor/Dockerfile +++ b/src/DataProcessor/Dockerfile @@ -1,8 +1,8 @@ -FROM 127.0.0.1:5000/base:latest +FROM ghcr.io/h3xitsec/h3xrecon_base:latest -COPY ./DataProcessor /app/DataProcessor +COPY . /app COPY ./QueueManager /app/QueueManager ENV NAME=DataProcessor -CMD ["python", "./DataProcessor/start_data_processor.py"] +CMD ["python", "start_data_processor.py"] diff --git a/src/JobProcessor/Dockerfile b/src/JobProcessor/Dockerfile index f905b6d..fd2e208 100644 --- a/src/JobProcessor/Dockerfile +++ b/src/JobProcessor/Dockerfile @@ -1,8 +1,8 @@ -FROM 127.0.0.1:5000/base:latest +FROM ghcr.io/h3xitsec/h3xrecon_base:latest -COPY ./JobProcessor /app/JobProcessor +COPY . /app COPY ./QueueManager /app/QueueManager ENV NAME=JobProcessor -CMD ["python", "./JobProcessor/start_job_processor.py"] +CMD ["python", "start_job_processor.py"] diff --git a/src/Logger/Dockerfile b/src/Logger/Dockerfile index c6e8a64..d74a63f 100644 --- a/src/Logger/Dockerfile +++ b/src/Logger/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim +FROM ghcr.io/h3xitsec/h3xrecon_base:latest WORKDIR /app ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache @@ -7,7 +7,9 @@ COPY ./requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r requirements.txt # Copy application code -COPY ./Logger /app/Logger +COPY . /app +COPY ./DatabaseManager /app/DatabaseManager +COPY ./QueueManager /app/QueueManager # Command to run the application CMD ["python", "Logger.py"] \ No newline at end of file diff --git a/src/Worker/Dockerfile b/src/Worker/Dockerfile index 43d0895..5cd4313 100644 --- a/src/Worker/Dockerfile +++ b/src/Worker/Dockerfile @@ -1,31 +1,10 @@ -# Use an official Python runtime as a parent image -FROM python:3.9.16-slim - -WORKDIR /app - -# Install additional system dependencies -RUN apt-get update && apt-get install -y \ - dnsutils \ - && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt /app/requirements.txt - -ENV PYTHONPATH="/app" - -RUN pip install \ - --upgrade \ - --trusted-host pypi.org \ - --trusted-host pypi.python.org \ - --trusted-host files.pythonhosted.org \ - --no-cache-dir \ - -r requirements.txt +FROM ghcr.io/h3xitsec/h3xrecon_base:latest ENV PATH="$PATH:/root/go/bin:/root/.pdtm/go/bin:/usr/lib/go-1.15/bin" -# Install additional system dependencies RUN apt-get update && apt-get install -y \ git \ - golang-1.15 \ + golang-go \ jq \ prips \ nmap \ @@ -35,10 +14,11 @@ RUN apt-get update && apt-get install -y \ RUN pip install dnspython -RUN mkdir -p /root/.pdtm/go/bin && \ - wget https://github.com/projectdiscovery/pdtm/releases/download/v0.0.9/pdtm_0.0.9_linux_arm64.zip -O /tmp/pdtm.zip &&\ - unzip /tmp/pdtm.zip && \ - mv pdtm /root/.pdtm/go/bin/pdtm && \ +#RUN mkdir -p /root/.pdtm/go/bin && \ +# wget https://github.com/projectdiscovery/pdtm/releases/download/v0.0.9/pdtm_0.0.9_linux_arm64.zip -O /tmp/pdtm.zip &&\ +# unzip /tmp/pdtm.zip && \ +# mv pdtm /root/.pdtm/go/bin/pdtm && \ +RUN go install -v github.com/projectdiscovery/pdtm/cmd/pdtm@latest && \ pdtm -i subfinder && \ pdtm -i httpx && \ pdtm -i dnsx @@ -47,16 +27,12 @@ RUN git clone https://github.com/UnaPibaGeek/ctfr.git /opt/ctfr && \ cd /opt/ctfr && \ pip install -r requirements.txt -ENV GOROOT=/usr/lib/go-1.15 -ENV GO111MODULE=on - -RUN go get github.com/Josue87/gotator@latest +RUN go install github.com/Josue87/gotator@latest -COPY ./Worker /Worker/app -COPY ./QueueManager /app/QueueManager +COPY . /app COPY ./DatabaseManager /app/DatabaseManager +COPY ./QueueManager /app/QueueManager ENV NAME=Worker -# Run worker.py when the container launches -CMD ["python", "Worker/Worker/start_worker.py"] +CMD ["python", "start_worker.py"] diff --git a/src/docker-compose.processor.yaml b/src/docker-compose.processor.yaml deleted file mode 100644 index fccd496..0000000 --- a/src/docker-compose.processor.yaml +++ /dev/null @@ -1,179 +0,0 @@ -name: h3xrecon_processor -services: - baseimage: - build: ./BaseImage - image: h3xrecon/base - - tailscale: - image: tailscale/tailscale:latest - environment: - - TS_AUTH_KEY_FILE=/run/secrets/tailscale_auth_key - - TS_AUTH_KEY= - - TS_HOSTNAME=processor - - TS_STATE_DIR=/var/lib/tailscale - - TS_EXTRA_ARGS=--reset - - TS_USERSPACE=false - - TS_SERVE_CONFIG=./serve.yaml - - TS_SERVE_PORT_RANGES=4222,5432,6379 - volumes: - - tailscale_state:/var/lib/tailscale - - /dev/net/tun:/dev/net/tun - - ./tailscale/serve.yaml:/var/lib/tailscale/serve.yaml - - ./tailscale/entrypoint.sh:/entrypoint.sh - cap_add: - - net_admin - - sys_module - secrets: - - tailscale_auth_key - restart: unless-stopped - entrypoint: ["/bin/sh", "/entrypoint.sh"] - # logging: - # driver: "none" - privileged: true - healthcheck: - test: ["CMD", "tailscale", "status"] - interval: 10s - timeout: 5s - retries: 3 - - redis: - image: redis:6.2-alpine - network_mode: service:tailscale - volumes: - - redis_data:/data - command: redis-server --appendonly yes --protected-mode no - depends_on: - tailscale: - condition: service_healthy - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 5s - timeout: 3s - retries: 5 - - pgsql: - environment: - - POSTGRES_PASSWORD= - - POSTGRES_PASSWORD_FILE=/run/secrets/postgresql_db_password - - POSTGRES_USER=h3xrecon - - POSTGRES_DB=h3xrecon - build: - context: ./pgsql - dockerfile: Dockerfile - secrets: - - postgresql_db_password - network_mode: service:tailscale - volumes: - - pgsql_data:/var/lib/postgresql/data - - ./pgsql/psql_dump.sql:/docker-entrypoint-initdb.d/psql_dump.sql - #hostname: pgsql-server - depends_on: - tailscale: - condition: service_healthy - healthcheck: - test: ["CMD-SHELL", "pg_isready -U h3xrecon -d h3xrecon"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - - nats: - build: - context: ./nats - dockerfile: Dockerfile - network_mode: service:tailscale - #hostname: nats-server - volumes: - - nats_data:/data - - ./nats/init.sh:/init.sh - depends_on: - tailscale: - condition: service_healthy - healthcheck: - test: nats stream ls --server=nats://localhost:4222 | grep -q "FUNCTION_EXECUTE" && nats stream ls --server=nats://localhost:4222 | grep -q "FUNCTION_OUTPUT" && nats stream ls --server=nats://localhost:4222 | grep -q "RECON_DATA" - interval: 2s - timeout: 2s - retries: 5 - start_period: 5s - - logger: - env_file: .env - build: - context: ./Logger - dockerfile: Dockerfile - secrets: - - postgresql_db_password - network_mode: service:tailscale - volumes: - - ./Logger:/app - depends_on: - tailscale: - condition: service_healthy - nats: - condition: service_healthy - pgsql: - condition: service_healthy - environment: - - LOGURU_LEVEL=DEBUG - - job_processor: - env_file: .env - build: - context: ./ - dockerfile: ./JobProcessor/Dockerfile - secrets: - - postgresql_db_password - network_mode: service:tailscale - #hostname: job-processor - volumes: - - ./JobProcessor:/app/JobProcessor - - ./QueueManager:/app/QueueManager - depends_on: - tailscale: - condition: service_healthy - nats: - condition: service_healthy - redis: - condition: service_healthy - environment: - - LOGURU_LEVEL=DEBUG - - deploy: - mode: replicated - replicas: 1 - - data_processor: - env_file: .env - build: - context: ./ - dockerfile: ./DataProcessor/Dockerfile - secrets: - - postgresql_db_password - network_mode: service:tailscale - volumes: - - ./DataProcessor:/app/DataProcessor - - ./QueueManager:/app/QueueManager - depends_on: - tailscale: - condition: service_healthy - nats: - condition: service_healthy - pgsql: - condition: service_healthy - environment: - - LOGURU_LEVEL=DEBUG - deploy: - mode: replicated - replicas: 1 - -volumes: - pgsql_data: {} - redis_data: {} - nats_data: {} - tailscale_state: {} - -secrets: - tailscale_auth_key: - file: ./secrets/tailscale_auth_key_processor.txt - postgresql_db_password: - file: ./secrets/postgresql_db_password.txt \ No newline at end of file diff --git a/src/docker-compose.workers.yaml b/src/docker-compose.workers.yaml deleted file mode 100644 index 20ef1d7..0000000 --- a/src/docker-compose.workers.yaml +++ /dev/null @@ -1,66 +0,0 @@ -name: h3xrecon_worker -services: - baseimage: - build: ./BaseImage - image: h3xrecon/base - - tailscale: - env_file: .env - image: tailscale/tailscale:latest - environment: - - TS_AUTH_KEY_FILE=/run/secrets/tailscale_auth_key - - TS_AUTH_KEY= - - TS_HOSTNAME=worker - - TS_STATE_DIR=/var/lib/tailscale - - TS_EXTRA_ARGS=--reset - - TS_USERSPACE=false - volumes: - - tailscale_state:/var/lib/tailscale - - /dev/net/tun:/dev/net/tun - - ./tailscale/entrypoint.sh:/entrypoint.sh - secrets: - - tailscale_auth_key - cap_add: - - net_admin - - sys_module - restart: unless-stopped - entrypoint: ["/bin/sh", "/entrypoint.sh"] - privileged: true - healthcheck: - test: ["CMD", "tailscale", "status"] - interval: 10s - timeout: 5s - retries: 3 - - worker: - env_file: .env - build: - context: ./ - dockerfile: ./Worker/Dockerfile - network_mode: service:tailscale - volumes: - - ./Worker:/app/Worker - - ./QueueManager:/app/QueueManager - - ./DatabaseManager:/app/DatabaseManager - depends_on: - tailscale: - condition: service_healthy - restart: true - baseimage: - condition: service_completed_successfully - environment: - - LOGURU_LEVEL=DEBUG - secrets: - - postgresql_db_password - deploy: - mode: replicated - replicas: 5 - -volumes: - tailscale_state: {} - -secrets: - tailscale_auth_key: - file: ./secrets/tailscale_auth_key_worker.txt - postgresql_db_password: - file: ./secrets/postgresql_db_password.txt \ No newline at end of file diff --git a/src/nats/Dockerfile b/src/nats/Dockerfile index 23d4953..cc93a00 100644 --- a/src/nats/Dockerfile +++ b/src/nats/Dockerfile @@ -10,8 +10,8 @@ RUN wget https://github.com/nats-io/natscli/releases/download/v0.1.5/nats-0.1.5- && rm -rf nats-0.1.5-linux-amd64* # Copy scripts -COPY init.sh /init.sh -COPY entrypoint.sh /entrypoint.sh +COPY ./init.sh /init.sh +COPY ./entrypoint.sh /entrypoint.sh RUN chmod +x /init.sh /entrypoint.sh CMD ["/entrypoint.sh"] \ No newline at end of file diff --git a/src/pgsql/Dockerfile b/src/pgsql/Dockerfile index 84e5195..6fd0881 100644 --- a/src/pgsql/Dockerfile +++ b/src/pgsql/Dockerfile @@ -2,4 +2,4 @@ FROM postgres ENV POSTGRES_DB=h3xrecon ENV POSTGRES_PASSWORD_FILE=/run/secrets/postgresql_db_password ENV POSTGRES_USER=h3xrecon -COPY psql_dump.sql /docker-entrypoint-initdb.d/ +COPY ./psql_dump.sql /docker-entrypoint-initdb.d/ diff --git a/src/pgsql/psql_dump.sql b/src/pgsql/psql_dump.sql index 089d24a..9d7e136 100644 --- a/src/pgsql/psql_dump.sql +++ b/src/pgsql/psql_dump.sql @@ -244,38 +244,12 @@ $$; -- Insert the program 'h3xit' INSERT INTO programs (name) VALUES ('h3xit') ON CONFLICT (name) DO NOTHING; -INSERT INTO programs (name) VALUES ('desjardins') ON CONFLICT (name) DO NOTHING; INSERT INTO programs (name) VALUES ('test') ON CONFLICT (name) DO NOTHING; -INSERT INTO program_cidrs (program_id, cidr) VALUES -((SELECT id FROM programs WHERE name = 'desjardins'), '142.195.0.0/16'); - -- Insert the scope regexes for 'h3xit INSERT INTO program_scopes (program_id, regex) VALUES ((SELECT id FROM programs WHERE name = 'h3xit'), '(.*\.)?h3x\.it$'), ((SELECT id FROM programs WHERE name = 'h3xit'), '(.*\.)?h3xit\.io$'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*desjardins.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*dsf-dfs.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*duproprio.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*disnat.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*bonidollar.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*bonusdollars.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*lapersonnelle.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*desjardinsbank.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*fondsdesjardins.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*desjardinsinsurance.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*desjardinsassurancevie.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*desjardinsgeneralinsurances.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*desjardinsassurancesgenerales.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*duproprio.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*hexavest.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*sfl.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*dfsi.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*investissementdesjardins.*'), -((SELECT id FROM programs WHERE name = 'desjardins'), '.*sflplacement.*'), --- ((SELECT id FROM programs WHERE name = 'desjardins'), '(.*\.)?desjardins\.com$'), --- ((SELECT id FROM programs WHERE name = 'desjardins'), '(.*\.)?dsf-dfs\.com$'), --- ((SELECT id FROM programs WHERE name = 'desjardins'), '(.*\.)?duproprio\.com$'), ((SELECT id FROM programs WHERE name = 'test'), '(.*\.)?test\.com$'), ((SELECT id FROM programs WHERE name = 'test'), '(.*\.)?example\.com$'), ((SELECT id FROM programs WHERE name = 'test'), '(.*\.)?example\.net$'); diff --git a/syncvps.sh b/syncvps.sh deleted file mode 100755 index 8001b1d..0000000 --- a/syncvps.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/etc/profiles/per-user/h3x/bin/bash - -rsync -av /mnt/data/projects/h3xrecon/build/Processor recon:/home/h3x/h3xrecon/ --exclude=.git --exclude="**/__pycache__/*" --exclude=venv - -#rsync -av /mnt/data/projects/h3xrecon_v2/ vps4:/home/ubuntu/h3xrecon_v2/ --exclude=.git --exclude="**/__pycache__/*" --exclude=venv \ No newline at end of file