Fix/fetch executor information

.circleci/config.yml

@@ -0,0 +1,57 @@
+version: 2.1
+
+orbs:
+  secops: apollo/[email protected]
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - test
+  security-scans:
+    jobs:
+      - secops/gitleaks:
+          context:
+            - platform-docker-ro
+            - github-orb
+            - secops-oidc
+          git-base-revision: <<#pipeline.git.base_revision>><<pipeline.git.base_revision>><</pipeline.git.base_revision >>
+          git-revision: << pipeline.git.revision >>
+
+cache: &cache v0-{{ checksum "setup.py" }}
+
+jobs:
+  test:
+    docker:
+      - image: python:3.9.5-buster
+    working_directory: /code/
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - *cache
+
+      - run:
+          name: Install tap-circle-ci
+          command: |
+            python -m venv venv/tap-circle-ci
+            source venv/tap-circle-ci/bin/activate
+            pip install -e .[tests]
+            deactivate
+
+      - save_cache:
+          key: *cache
+          paths:
+            - "./venv"
+            - "/usr/local/bin"
+            - "/usr/local/lib/python3.9/site-packages"
+
+      - run:
+          name: Run Tests
+          command: |
+            source venv/tap-circle-ci/bin/activate
+            pytest --verbose
+
+      - store_artifacts:
+          path: target/test-results
+          destination: raw-test-output

.gitignore

@@ -2,5 +2,7 @@
 *.egg
 __pycache__/
 
-sample_config.json
+venv/
 catalog.json
+config.json
+state.json

CODEOWNERS

@@ -0,0 +1,4 @@
+# This file was automatically generated by the Apollo SecOps team
+# Please customize this file as needed prior to merging.
+
+* @apollographql/data

README.md

@@ -1,5 +1,26 @@
-# tap-circle-ci
-
+# tap-circleci
+
+## Meltano Extractor Setup
+
+```
+meltano add extractor --custom tap-circleci
+```
+In the interactive portion, use these variables
+```
+name: tap-circleci
+pypi: git+https://github.com/JChouCode/tap-circleci.git
+executable: tap-circleci
+capabilities: discover,catalog,state
+config: project_slugs,token:password
+```
+
+## Improvements
+This fork improves the tap to handle edge cases that cause errors.
+- Edge Case: Job is cancelled and build number is not created, causing a 404 error when requesting unknown build number.
+- Improved Bookmarking
+- Added `tooling/` for various scripts which help wrangle some of the sharp corners of CircleCI
+
+## Sisu Data - About
 This is a [Singer](https://singer.io) tap that produces JSON-formatted data
 following the [Singer
 spec](https://github.com/singer-io/getting-started/blob/master/SPEC.md).
@@ -8,94 +29,134 @@ This tap:
 
 - Pulls raw data from [Circle CI](https://circleci.com/)
 - Extracts the following resources:
-  - [Pipelines](https://circleci.com/docs/api/v2/#get-all-pipelines)
-  - [Workflows](https://circleci.com/docs/api/v2/#get-a-pipeline-39-s-workflows)
-  - [Jobs](https://circleci.com/docs/api/v2/#get-a-workflow-39-s-jobs)
+  - [Pipelines](hhttps://circleci.com/docs/api/v2/#operation/listPipelines)
+  - [Workflows](https://circleci.com/docs/api/v2/#operation/listWorkflowsByPipelineId)
+  - [Jobs](https://circleci.com/docs/api/v2/#operation/listWorkflowJobs)
+  - [Steps](https://circleci.com/docs/api/#single-job)
 - Outputs the schema for each resource
 - Incrementally pulls data based on the input state
 
-
 ## Quick start
 
 1. Install
 
-    ```bash
-    git clone [email protected]:sisudata/tap-circle-ci.git && cd tap-circle-ci && pip install -e .
-    ```
+   ```bash
+   git clone [email protected]:apollographql/tap-circleci.git && cd tap-circleci && pip install -e .
+   ```
 
 2. Create a Circle CI access token
 
-    Login to your Circle CI account, go to the
-    [Personal API Tokens](https://circleci.com/account/api)
-    page, and generate a new token. Copy the token and save it somewhere safe.
+   Login to your Circle CI account, go to the
+   [Personal API Tokens](https://circleci.com/account/api)
+   page, and generate a new token. Copy the token and save it somewhere safe.
 
-3. Create the config file
+3. Create the config file (see below)
 
-    Create a JSON file containing the token you just created as well as the project slug to the project you want to extract data from. Retrieve the project slug
-    from the url for a workflow - it should be the VCS your project uses (`gh` for Github or `bb` for Bitbucket), followed by the owner or organization, followed by the repository name
-    ex. `gh/singer-io/singer-python`. You can enter multiple project slugs separated by spaces to pull data from multiple projects.
+   Create a JSON file containing the token you just created as well as the project slug to the project you want to extract data from. Retrieve the project slug
+   from the url for a workflow - it should be the VCS your project uses (`gh` for Github or `bb` for Bitbucket), followed by the owner or organization, followed by the repository name
+   ex. `gh/singer-io/singer-python`. You can enter multiple project slugs separated by spaces to pull data from multiple projects.
+
+   ```json
+   {
+     "token": "your-access-token",
+     "project_slugs": "gh/singer-io/singer-python gh/singer-io/getting-started"
+   }
+   ```
 
-    ```json
-    {
-      "token": "your-access-token",
-      "project_slugs": "gh/singer-io/singer-python gh/singer-io/getting-started"
-    }
-    ```
 4. Run the tap in discovery mode to get catalog.json file
 
-    ```bash
-    tap-circle-ci --config config.json --discover > catalog.json
-    ```
+   ```bash
+   tap-circleci --config config.json --discover > catalog.json
+   ```
+
 5. In the catalog.json file, select the streams to sync
 
-    Each stream in the properties.json file has a "metadata" entry.  To select a stream to sync, add
-    `{"breadcrumb": [], "metadata": {"selected": true}}` to that stream's "metadata" entry.  
-    For example, to sync the pipelines stream:
-    ```
-    ...
-        "type": [
-          "null",
-          "object"
-        ],
-        "additionalProperties": false
-      },
-      "stream": "pipelines",
-      "metadata": [{"breadcrumb": [], "metadata": {"selected": true}}]
-    },
-    ...
-    ```
-    Another way to select a stream to sync is to add `"selected": true` into that stream's schema:
-
-    ```
-    ...
-    "tap_stream_id": "workflows",
-    "key_properties": [],
-    "schema": {
-      "selected": true,
-      "properties": {
-        "_pipeline_id": {
-          "type": [
-            "null",
-            "string"
-          ]
-    ...
-    ```
-    Either way is acceptable, but the first way is preferred.
+   Each stream in the properties.json file has a "metadata" entry. To select a stream to sync, add
+   `{"breadcrumb": [], "metadata": {"selected": true}}` to that stream's "metadata" entry.  
+   For example, to sync the pipelines stream:
+
+   ```
+   ...
+       "type": [
+         "null",
+         "object"
+       ],
+       "additionalProperties": false
+     },
+     "stream": "pipelines",
+     "metadata": [{"breadcrumb": [], "metadata": {"selected": true}}]
+   },
+   ...
+   ```
+
+   Another way to select a stream to sync is to add `"selected": true` into that stream's schema:
+
+   ```
+   ...
+   "tap_stream_id": "workflows",
+   "key_properties": [],
+   "schema": {
+     "selected": true,
+     "properties": {
+       "_pipeline_id": {
+         "type": [
+           "null",
+           "string"
+         ]
+   ...
+   ```
+
+   Either way is acceptable, but the first way is preferred.
 
 6. Run the application (will print records and other messages to the console)
 
-    `tap-circle-ci` can be run with:
+   `tap-circleci` can be run with:
+
+   ```bash
+   tap-circleci --config config.json --catalog catalog.json
+   ```
+
+   To save output to a file:
+
+   ```bash
+   tap-circleci --config config.json --catalog catalog.json > output.txt
+   ```
+
+   It is our intention that this singer tap gets used with a singer target, which will load the output into a database.
+   More information on singer targets [here](https://github.com/singer-io/getting-started/blob/master/docs/RUNNING_AND_DEVELOPING.md#running-a-singer-tap-with-a-singer-target).
+
+7. To rerun using the last output `STATE` record:
+
+   In your output records, you will see something like:
+
+   ```json
+   {
+     "type": "STATE",
+     "value": {
+       "bookmarks": {
+         "gh/apollographql/tap-circleci": {
+           "pipelines": { "since": "2023-11-15T00:00:00.000000Z" }
+         }
+       }
+     }
+   }
+   ```
+
+   Select the `value` key, store it to a JSON file, and run:
+
+   ```bash
+   tap-circleci --config config.json --catalog catalog.json --state state.json
+   ```
+
+## Configuration
+
+Detailed configuration information for the `--config` key.
 
-    ```bash
-    tap-circle-ci --config config.json --catalog catalog.json
-    ```
+| key             | type     | default | description                                            |
+| --------------- | -------- | ------- | ------------------------------------------------------ |
+| `token`         | `string` | `N/A`   | [Personal API Token](https://circleci.com/account/api) |
+| `project_slugs` | `string` | `N/A`   | Space ` ` delimited string of CCI project slugs        |
 
-    To save output to a file:
-    ```bash
-    tap-circle-ci --config config.json --catalog catalog.json > output.txt
-    ```
-    It is our intention that this singer tap gets used with a singer target, which will load the output into a database.
-    More information on singer targets [here](https://github.com/singer-io/getting-started/blob/master/docs/RUNNING_AND_DEVELOPING.md#running-a-singer-tap-with-a-singer-target).
 ---
 
 Copyright © 2020 Sisu Data

docker-compose.yml

@@ -0,0 +1,10 @@
+version: '3'
+services:
+  tap:
+    image: python:3.9.5-buster
+    entrypoint: /code/docker-entrypoint.sh
+    working_dir: /code
+    environment:
+      DEPLOYMENT: "${DEPLOYMENT}"
+    volumes:
+      - .:/code

docker-entrypoint.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+python -m venv venv/tap-circle-ci
+source /code/venv/tap-circle-ci/bin/activate
+
+pip install -e .[tests]
+
+echo "source /code/venv/tap-circle-ci/bin/activate" >> ~/.bashrc
+echo -e "\n\nINFO: Dev environment ready."
+
+tail -f /dev/null

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+filterwarnings =
+    error
+    ignore::UserWarning
+    ignore:.*Using or importing the ABCs from:DeprecationWarning

sample-config.json

@@ -0,0 +1,4 @@
+{
+    "token": "TODO",
+    "project_slugs": "gh/singer-io/singer-python gh/singer-io/getting-started gh/apollographql/tap-circleci"
+}

sample-state.json

@@ -0,0 +1,7 @@
+{
+  "bookmarks": {
+    "gh/apollographql/tap-circle-ci": {
+      "pipelines": { "since": "2023-11-15T00:00:00.000000Z" }
+    }
+  }
+}

setup.py

@@ -2,7 +2,7 @@
 from setuptools import setup
 
 setup(
-    name="tap-circle-ci",
+    name="tap-circleci",
     version="0.1.0",
     description="Singer.io tap for extracting data from circle ci",
     author="Sisu Data",
@@ -11,13 +11,16 @@
     py_modules=["tap_circle_ci"],
     install_requires=[
         "singer-python>=5.0.12",
-        "requests",
-        "pytest",
-        "mock"
+        "requests"
     ],
+    extras_require={
+        'tests': [
+            "mock",
+            "pytest"
+        ]},
     entry_points="""
     [console_scripts]
-    tap-circle-ci=tap_circle_ci:main
+    tap-circleci=tap_circle_ci:main
     """,
     packages=["tap_circle_ci"],
     package_data={