Merge branch 'master' of ghh3x:h3xitsec/h3xrecon

.github/workflows/build-push.yaml

@@ -0,0 +1,110 @@
+name: Build and Push Docker Images
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+permissions:
+  packages: write
+  contents: read
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run build script
+        run: bash build.sh
+
+      - name: Build and push base image
+        uses: docker/build-push-action@v4
+        with:
+          context: ./build/BaseImage
+          file: ./build/BaseImage/Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_base:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64,linux/arm64
+
+      - name: Build and push worker image
+        uses: docker/build-push-action@v4
+        with:
+          context: ./build/Worker
+          file: ./build/Worker/Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_worker:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64,linux/arm64
+
+      - name: Build and push data processor image
+        uses: docker/build-push-action@v4
+        with:
+          context: ./build/DataProcessor
+          file: ./build/DataProcessor/Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_dataprocessor:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64
+
+      - name: Build and push job processor image
+        uses: docker/build-push-action@v4
+        with:
+          context: ./build/JobProcessor
+          file: ./build/JobProcessor/Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_jobprocessor:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64
+
+      - name: Build and push logger image
+        uses: docker/build-push-action@v4
+        with:
+          context: ./build/Logger
+          file: ./build/Logger/Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_logger:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64
+
+      - name: Build and push nats image
+        uses: docker/build-push-action@v4
+        with:
+          context: ./build/nats
+          file: ./build/nats/Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_nats:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64
+
+      - name: Build and push pgsql image
+        uses: docker/build-push-action@v4
+        with:
+          context: ./build/pgsql
+          file: ./build/pgsql/Dockerfile
+          push: true
+          tags: ghcr.io/${{ github.repository_owner }}/h3xrecon_pgsql:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64

.gitignore

@@ -7,13 +7,15 @@ env.sh
 .env
 secrets/
 build/
+shell.nix
 vault.yaml
 tailscale_vault.yaml
+old_keep/
 *.key
 *.pub
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 
 # C extensions
-*.so
+*.so

Ansible/deploy_stack.yaml

@@ -0,0 +1,23 @@
+- name: Start h3xrecon stack
+  hosts: processor
+  vars_files:
+    - vault.yaml
+  tasks:
+    - delegate_to: localhost
+      ansible.builtin.copy:
+        dest: /tmp/h3xrecon_processor.key
+        content: "{{ PROCESSOR_PRIVATE_KEY }}"
+        mode: 0600
+    - name: Deploy Docker stack
+      delegate_to: localhost
+      community.docker.docker_stack:
+        state: present
+        name: h3xrecon
+        compose:
+          - ../deploy/docker-compose.yaml
+      environment:
+        DOCKER_HOST: ssh://{{ ansible_host }}
+    - delegate_to: localhost
+      ansible.builtin.file:
+        path: /tmp/h3xrecon_processor.key
+        state: absent

Ansible/remove_stack.yaml

@@ -0,0 +1,23 @@
+- name: Remove h3xrecon stack
+  hosts: processor
+  vars_files:
+    - vault.yaml
+  tasks:
+    - delegate_to: localhost
+      ansible.builtin.copy:
+        dest: /tmp/h3xrecon_processor.key
+        content: "{{ PROCESSOR_PRIVATE_KEY }}"
+        mode: 0600
+    - name: Deploy Docker stack
+      delegate_to: localhost
+      community.docker.docker_stack:
+        state: absent
+        name: h3xrecon
+        compose:
+          - ../deploy/docker-compose.yaml
+      environment:
+        DOCKER_HOST: ssh://{{ ansible_host }}
+    - delegate_to: localhost
+      ansible.builtin.file:
+        path: /tmp/h3xrecon_processor.key
+        state: absent

Ansible/setup_nodes.yaml

@@ -48,11 +48,11 @@
       path: "{{ h3xrecon_target_directory }}/"
       state: absent
 
-  - name: Sync codebase
-    ansible.builtin.synchronize:
-      src: "{{ h3xrecon_source_directory }}"
-      dest: "{{ h3xrecon_target_directory }}"
-      rsync_opts: '--exclude=.git --exclude="**/__pycache__/*" --exclude=venv'
+  # - name: Sync codebase
+  #   ansible.builtin.synchronize:
+  #     src: "{{ h3xrecon_source_directory }}"
+  #     dest: "{{ h3xrecon_target_directory }}"
+  #     rsync_opts: '--exclude=.git --exclude="**/__pycache__/*" --exclude=venv'
 
   - name: Setup Python Virtual Environment
     ansible.builtin.pip:
@@ -67,9 +67,9 @@
       groups: docker
       append: yes
 
-  - name: Enable arm64 image building
-    become: true
-    ansible.builtin.shell: docker run --privileged --rm tonistiigi/binfmt --install all
+  # - name: Enable arm64 image building
+  #   become: true
+  #   ansible.builtin.shell: docker run --privileged --rm tonistiigi/binfmt --install all
 
-  - name: Build Images
-    ansible.builtin.include_tasks: tasks/build_container_images.yaml
+  # - name: Build Images
+  #   ansible.builtin.include_tasks: tasks/build_container_images.yaml