From ba6835f487a17a5641c173d95a422e213bdabb82 Mon Sep 17 00:00:00 2001 From: Luigi Marini Date: Mon, 12 Feb 2024 15:59:56 -0600 Subject: [PATCH] Pre commit autoformat (#846) * Removed unused imports using autoflake. * Ordered imports using isort. * Reformatted using black. * Setting up pre-commits. * Ran pre-commit an all files. * Working on creating openapi.json without running backend to be used with codegen. * Fixed `MUI: The Tabs component doesn't accept a Fragment as a child.` * Fixed runtime error due to replacing `== None` with 'is None'. * Fixed runtime error due to replacing `== None` with 'is None'. * Automatically generate openapi json file using pre-commit hook. * Added codegen precommit hook. * Setting up prettier and eslint. * First run of prettier. * Runnin eslint and prettier as hooks. * Skip codegen directories when running eslint and prettier. * Flake8 E711 exception. Required for Beanie query to work. * Fixed search_users_prefix codegen stub. * Fixed beanie query for get_dataset_folders. * Ignore Flake8 E711 since beanie query do not support `is None`. --- .flake8 | 5 + .github/workflows/backend.yaml | 2 +- .github/workflows/codegen.yaml | 2 +- .github/workflows/helm-release.yaml | 2 +- .pre-commit-config.yaml | 53 + .run/Clowder2 Dev.run.xml | 2 +- .run/Debug.run.xml | 2 +- .run/Docker Dev.run.xml | 2 +- .run/Python tests in tests.run.xml | 2 +- .run/start.run.xml | 2 +- .run/start_dev.run.xml | 2 +- .run/uvicorn.run.xml | 2 +- Pipfile | 3 + Pipfile.lock | 231 +- README.md | 24 +- backend/Pipfile | 6 + backend/Pipfile.lock | 1873 ++- backend/app/config.py | 2 +- backend/app/database/errors.py | 5 +- backend/app/dependencies.py | 5 +- backend/app/deps/authorization_deps.py | 16 +- backend/app/heartbeat_listener_sync.py | 12 +- backend/app/keycloak_auth.py | 12 +- backend/app/logging.conf | 2 +- backend/app/main.py | 48 +- backend/app/models/datasets.py | 9 +- backend/app/models/errors.py | 5 +- backend/app/models/feeds.py | 8 +- backend/app/models/files.py | 7 +- backend/app/models/folder_and_file.py | 9 +- backend/app/models/folders.py | 7 +- backend/app/models/groups.py | 7 +- backend/app/models/listeners.py | 7 +- backend/app/models/metadata.py | 17 +- .../models/migrations/rename_dataset_field.py | 8 +- backend/app/models/readme.md | 1 - backend/app/models/search.py | 2 +- backend/app/models/status.py | 3 +- backend/app/models/thumbnails.py | 7 +- backend/app/models/tokens.py | 3 +- backend/app/models/users.py | 2 +- backend/app/models/visualization_config.py | 9 +- backend/app/models/visualization_data.py | 5 +- backend/app/rabbitmq/__init__.py | 1 - backend/app/rabbitmq/listeners.py | 7 +- backend/app/routers/authentication.py | 12 +- backend/app/routers/authorization.py | 23 +- backend/app/routers/datasets.py | 98 +- backend/app/routers/elasticsearch.py | 8 +- backend/app/routers/feeds.py | 23 +- backend/app/routers/files.py | 49 +- backend/app/routers/folders.py | 8 +- backend/app/routers/groups.py | 19 +- backend/app/routers/jobs.py | 19 +- backend/app/routers/keycloak.py | 29 +- backend/app/routers/listeners.py | 21 +- backend/app/routers/metadata.py | 21 +- backend/app/routers/metadata_datasets.py | 40 +- backend/app/routers/metadata_files.py | 46 +- backend/app/routers/public_datasets.py | 44 +- backend/app/routers/public_elasticsearch.py | 2 +- backend/app/routers/public_files.py | 61 +- backend/app/routers/public_folders.py | 8 +- backend/app/routers/public_metadata.py | 25 +- backend/app/routers/public_visualization.py | 38 +- backend/app/routers/status.py | 3 +- backend/app/routers/thumbnails.py | 22 +- backend/app/routers/users.py | 19 +- backend/app/routers/visualization.py | 32 +- backend/app/search/connect.py | 6 +- backend/app/search/index.py | 18 +- backend/app/tests/conftest.py | 4 +- backend/app/tests/test_apikey.py | 3 +- backend/app/tests/test_authorization.py | 11 +- backend/app/tests/test_datasets.py | 13 +- backend/app/tests/test_elastic_search.py | 9 +- backend/app/tests/test_extractors.py | 7 +- backend/app/tests/test_feeds.py | 7 +- backend/app/tests/test_files.py | 5 +- backend/app/tests/test_folders.py | 7 +- backend/app/tests/test_groups.py | 4 +- backend/app/tests/test_metadata.py | 3 +- backend/app/tests/test_visualization.py | 5 +- backend/app/tests/utils.py | 5 +- backend/extract-openapi.py | 48 + backend/heartbeat_listener.py | 15 +- backend/message_listener.py | 15 +- deployments/docker/minio-nginx.conf | 4 +- deployments/kubernetes/charts/README.md | 1 - .../kubernetes/charts/clowder2/README.md | 1 - .../charts/clowder2/templates/theme.yml | 2 +- .../kubernetes/charts/clowder2/values.yaml | 4 +- deployments/kubernetes/charts/radiant.yaml | 1 - docker-compose.dev.yml | 2 +- docker-dev.sh | 4 +- docs/README.md | 2 +- docs/source/architecture.md | 2 +- docs/source/conf.py | 4 +- docs/source/configure-keycloak-realm.md | 36 +- docs/source/dev-get-started.md | 35 +- docs/source/elasticsearch.md | 10 +- docs/source/index.rst | 2 +- docs/source/listeners.md | 50 +- docs/source/metadata.md | 40 +- docs/source/mongo-fastapi.md | 2 +- docs/source/visualizations.md | 2 +- frontend/.editorconfig | 2 +- frontend/.eslintrc | 12 +- frontend/.prettierignore | 1 + frontend/.prettierrc | 1 + frontend/clowder.conf | 1 - frontend/dependencies.txt | 1 - frontend/docker.sh | 8 +- frontend/package.json | 1 + frontend/prettierignore | 4 + frontend/release.sh | 2 +- frontend/src/actions/file.js | 4 +- frontend/src/actions/public_file.js | 22 +- frontend/src/actions/public_visualization.js | 8 +- frontend/src/actions/user.js | 2 +- frontend/src/actions/visualization.js | 2 +- frontend/src/components/Paragraph.tsx | 61 +- frontend/src/components/apikeys/ApiKey.tsx | 4 +- frontend/src/components/auth/AuthWrapper.tsx | 12 +- .../src/components/auth/RedirectLogout.tsx | 19 +- .../src/components/auth/RedirectRegister.tsx | 10 +- .../src/components/auth/legacy/Logout.tsx | 18 +- .../src/components/datasets/DatasetCard.tsx | 70 +- .../src/components/datasets/DatasetName.tsx | 83 +- .../datasets/EditDescriptionModal.tsx | 65 +- frontend/src/components/datasets/EditMenu.tsx | 71 +- .../components/datasets/EditStatusModal.tsx | 157 +- .../src/components/datasets/OtherMenu.tsx | 7 +- .../components/datasets/PublicDatasetCard.tsx | 70 +- .../src/components/dialog/ActionModal.tsx | 50 +- frontend/src/components/errors/Forbidden.tsx | 48 +- .../src/components/errors/PageNotFound.tsx | 48 +- frontend/src/components/files/File.tsx | 29 +- frontend/src/components/files/FileMenu.tsx | 32 +- frontend/src/components/files/FileSearch.tsx | 9 +- .../components/files/FilesTableFileEntry.tsx | 6 +- frontend/src/components/files/PublicFile.tsx | 2 +- .../files/PublicFileActionsMenu.tsx | 8 +- frontend/src/components/files/UploadFile.tsx | 2 +- .../components/files/UploadFileMultiple.tsx | 2 +- frontend/src/components/groups/Groups.tsx | 2 +- .../src/components/groups/MembersTable.tsx | 2 +- .../groups/MembersTableUserEntry.tsx | 4 +- .../listeners/ExtractionHistoryTab.tsx | 119 +- .../components/listeners/ExtractionJobs.tsx | 2 +- .../listeners/ExtractionJobsTableHeader.tsx | 13 +- .../metadata/CreateMetadataDefinition.tsx | 28 +- .../components/metadata/DisplayMetadata.tsx | 2 +- .../metadata/MetadataDefinitions.tsx | 4 +- .../widgets/MetadataDateTimePicker.tsx | 118 +- .../metadata/widgets/MetadataDeleteButton.tsx | 57 +- .../metadata/widgets/MetadataEditButton.tsx | 124 +- .../metadata/widgets/MetadataSelect.tsx | 7 +- .../metadata/widgets/MetadataTextField.tsx | 104 +- .../src/components/search/InputSearchBox.tsx | 73 +- .../styledComponents/ClowderButton.jsx | 8 +- .../styledComponents/ClowderInputLabel.jsx | 6 +- .../ClowderMetadataFormHelperText.jsx | 11 +- .../ClowderRjsfSelectWidget.jsx | 8 +- .../ClowderRjsfTextWidget.jsx | 4 +- .../styledComponents/ClowderTitle.jsx | 8 +- frontend/src/components/tabs/TabComponent.tsx | 8 +- .../versions/FileVersionHistory.tsx | 21 +- .../src/components/versions/VersionChip.tsx | 3 +- .../components/visualizations/Audio/Audio.tsx | 8 +- .../visualizations/Audio/manifest.json | 5 +- .../src/components/visualizations/CSV/CSV.tsx | 10 +- .../visualizations/CSV/manifest.json | 7 +- .../visualizations/Demo/manifest.json | 4 +- .../components/visualizations/Html/Html.tsx | 28 +- .../visualizations/Html/manifest.json | 4 +- .../components/visualizations/Image/Image.tsx | 4 +- .../visualizations/PublicVisualization.tsx | 41 +- .../PublicVisualizationCard.tsx | 2 +- .../PublicVisualizationRawBytesCard.tsx | 3 +- .../components/visualizations/Text/Text.tsx | 16 +- .../visualizations/Text/manifest.json | 4 +- .../visualizations/VegaSpec/VegaSpec.tsx | 33 +- .../visualizations/VegaSpec/manifest.json | 7 +- .../visualizations/Video/manifest.json | 5 +- frontend/src/index.tsx | 11 +- frontend/src/openapi/index.ts | 2 +- frontend/src/openapi/v1/core/ApiError.ts | 28 +- .../src/openapi/v1/core/ApiRequestOptions.ts | 29 +- frontend/src/openapi/v1/core/ApiResult.ts | 12 +- .../src/openapi/v1/core/CancelablePromise.ts | 205 +- frontend/src/openapi/v1/core/OpenAPI.ts | 42 +- frontend/src/openapi/v1/core/request.ts | 454 +- frontend/src/openapi/v1/index.ts | 64 +- frontend/src/openapi/v1/models/Collection.ts | 10 +- frontend/src/openapi/v1/models/Comment.ts | 4 +- frontend/src/openapi/v1/models/Dataset.ts | 47 +- .../src/openapi/v1/models/ExtractorsLabel.ts | 10 +- frontend/src/openapi/v1/models/JSONLD.ts | 3 +- frontend/src/openapi/v1/models/License.ts | 31 +- frontend/src/openapi/v1/models/MiniUser.ts | 3 +- frontend/src/openapi/v1/models/Space.ts | 6 +- frontend/src/openapi/v1/models/Tags.ts | 6 +- frontend/src/openapi/v1/models/UUID.ts | 2 +- frontend/src/openapi/v1/models/binary.ts | 96 +- .../src/openapi/v1/services/AdminService.ts | 266 +- .../openapi/v1/services/CollectionsService.ts | 1502 +- .../openapi/v1/services/CurationsService.ts | 212 +- .../openapi/v1/services/DTexturesService.ts | 42 +- .../openapi/v1/services/DatasetsService.ts | 2936 ++-- .../openapi/v1/services/ExtractionsService.ts | 582 +- .../openapi/v1/services/ExtractorsService.ts | 200 +- .../src/openapi/v1/services/FilesService.ts | 2610 ++-- .../src/openapi/v1/services/FoldersService.ts | 138 +- .../src/openapi/v1/services/LogosService.ts | 299 +- .../openapi/v1/services/MetadataService.ts | 386 +- .../openapi/v1/services/PreviewsService.ts | 40 +- .../src/openapi/v1/services/ProxyService.ts | 356 +- .../openapi/v1/services/ReportingService.ts | 238 +- .../src/openapi/v1/services/SearchService.ts | 122 +- .../src/openapi/v1/services/SectionService.ts | 370 +- .../src/openapi/v1/services/ServiceService.ts | 102 +- .../src/openapi/v1/services/SpacesService.ts | 808 +- .../src/openapi/v2/services/AuthService.ts | 8 +- .../src/openapi/v2/services/UsersService.ts | 4 +- frontend/src/reducers/file.ts | 6 +- frontend/src/reducers/public_file.ts | 20 +- frontend/src/reducers/public_visualization.ts | 8 +- frontend/src/routes.tsx | 46 +- frontend/src/schema/datasetSchema.json | 5 +- frontend/src/schema/fileSchema.json | 32 +- frontend/src/schema/groupSchema.json | 5 +- frontend/src/store/configureStore.js | 19 +- frontend/src/styles/main.css | 23 +- frontend/src/theme.ts | 2 +- frontend/src/types/global.d.ts | 4 +- frontend/src/utils/common.js | 1 - frontend/src/utils/visualization.js | 2 +- frontend/src/visualization.config.ts | 6 +- frontend/src/webpack-public-path.js | 2 +- openapi.json | 11712 ++++++++++++++++ scripts/README.md | 2 +- .../populate_fake_data/populate_fake_data.py | 2 +- scripts/keycloak/README.md | 12 +- .../clowder-theme/login/theme.properties | 2 +- scripts/keycloak/clowder-theme/readme.adoc | 4 +- scripts/metadata/config/inputTypes.json | 4 +- scripts/metadata/config/widgetTypes.json | 2 +- .../definitions/alternativeTitle.json | 2 +- scripts/metadata/definitions/coordinates.json | 2 +- scripts/metadata/definitions/doi.json | 2 +- scripts/metadata/definitions/latLon.json | 2 +- scripts/metadata/definitions/time.json | 2 +- scripts/metadata/definitions/unit.json | 2 +- scripts/mongoviews/README.md | 4 +- scripts/mongoviews/datasets_view.js | 2 +- scripts/mongoviews/files_view.js | 2 +- scripts/mongoviews/folders_view.js | 2 +- .../mongoviews/listener_job_updates_view.js | 6 +- scripts/mongoviews/listener_jobs_view.js | 2 +- scripts/mongoviews/metadata_view.js | 2 +- 261 files changed, 20740 insertions(+), 8870 deletions(-) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml create mode 100644 backend/extract-openapi.py create mode 100644 frontend/.prettierignore create mode 100644 frontend/.prettierrc create mode 100644 frontend/prettierignore create mode 100644 openapi.json diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..81b4f92ca --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 180 +extend-ignore = E203, E266, E501, W503, E741, E711 +exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,venv/*,src/*,.rst,build +max-complexity=16 diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index d1699c175..fdd43a128 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -111,4 +111,4 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.messagesMeta.outputs.tags }} file: ./backend/messages.Dockerfile - labels: ${{ steps.messagesMeta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.messagesMeta.outputs.labels }} diff --git a/.github/workflows/codegen.yaml b/.github/workflows/codegen.yaml index 4b315c3f8..8f5cfa968 100644 --- a/.github/workflows/codegen.yaml +++ b/.github/workflows/codegen.yaml @@ -57,7 +57,7 @@ jobs: - name: Run Codegen working-directory: frontend run: npm run codegen:v2:test - + - name: Verify Changed files uses: tj-actions/verify-changed-files@v14 id: verify-changed-files diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml index 3b6f5eedb..3ec8c7cb5 100644 --- a/.github/workflows/helm-release.yaml +++ b/.github/workflows/helm-release.yaml @@ -50,7 +50,7 @@ jobs: - name: Publish to NCSA OpenSource if: | - github.event_name != 'pull_request' + github.event_name != 'pull_request' && github.repository == env.MAIN_REPO uses: bsord/helm-push@v4.1.0 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..fa2045e5e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,53 @@ +repos: + - repo: local + hooks: + - id: extract-openapi + name: extract-openapi + entry: pipenv run extract-openapi + language: system + pass_filenames: false + - repo: local + hooks: + - id: generate-frontend-api-client + name: generate-frontend-api-client + entry: bash -c 'cd frontend && npm run codegen:v2:file' + language: system + pass_filenames: false + - repo: local + hooks: + - id: eslint + name: eslint + entry: bash -c "cd frontend && npx eslint --fix src" + language: system + - repo: local + hooks: + - id: prettier + name: prettier + entry: bash -c "cd frontend && npx prettier src --write" + language: system + - repo: https://github.com/psf/black + rev: 23.11.0 + hooks: + - id: black + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + args: [ "--profile=black" ] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + exclude: deployments/kubernetes/charts + - id: end-of-file-fixer + exclude: 'openapi.json|frontend/src/openapi/v2/' + - id: trailing-whitespace + exclude: 'openapi.json|frontend/src/openapi/v2/' + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.6 + hooks: + - id: shellcheck + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 diff --git a/.run/Clowder2 Dev.run.xml b/.run/Clowder2 Dev.run.xml index 5416f68c2..c6cf599eb 100644 --- a/.run/Clowder2 Dev.run.xml +++ b/.run/Clowder2 Dev.run.xml @@ -5,4 +5,4 @@ - \ No newline at end of file + diff --git a/.run/Debug.run.xml b/.run/Debug.run.xml index d84dd7939..e35da2a88 100644 --- a/.run/Debug.run.xml +++ b/.run/Debug.run.xml @@ -2,4 +2,4 @@ - \ No newline at end of file + diff --git a/.run/Docker Dev.run.xml b/.run/Docker Dev.run.xml index 47e795b58..e60f643ae 100644 --- a/.run/Docker Dev.run.xml +++ b/.run/Docker Dev.run.xml @@ -10,4 +10,4 @@ - \ No newline at end of file + diff --git a/.run/Python tests in tests.run.xml b/.run/Python tests in tests.run.xml index 9a0097500..2fd8345de 100644 --- a/.run/Python tests in tests.run.xml +++ b/.run/Python tests in tests.run.xml @@ -14,4 +14,4 @@