diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 0000000..b0205fc --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,11 @@ +# Do not edit - changes here will be overwritten by Copier +_commit: v1 +_src_path: gh:pydev-guide/pyrepo-copier +author_email: alisterburt@gmail.com +author_name: Alister Burt +github_username: teamtomo +mode: tooling +module_name: starfile +project_name: starfile +project_short_description: STAR file I/O in Python + diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..f5b1733 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,15 @@ +* starfile version: +* Python version: +* Operating System: + +### Description + +Describe what you were trying to get done. +Tell us what happened, what went wrong, and what you expected to happen. + +### What I Did + +``` +Paste the command(s) you ran and the output. +If there was a crash, please include the traceback here. +``` diff --git a/.github/TEST_FAIL_TEMPLATE.md b/.github/TEST_FAIL_TEMPLATE.md new file mode 100644 index 0000000..3512972 --- /dev/null +++ b/.github/TEST_FAIL_TEMPLATE.md @@ -0,0 +1,12 @@ +--- +title: "{{ env.TITLE }}" +labels: [bug] +--- +The {{ workflow }} workflow failed on {{ date | date("YYYY-MM-DD HH:mm") }} UTC + +The most recent failing test was on {{ env.PLATFORM }} py{{ env.PYTHON }} +with commit: {{ sha }} + +Full run: https://github.com/{{ repo }}/actions/runs/{{ env.RUN_ID }} + +(This post will be updated if another test fails, as long as this issue remains open.) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..96505a9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "ci(dependabot):" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c1d4bc3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,108 @@ +name: CI + +on: + push: + branches: + - main + tags: + - "v*" + pull_request: + workflow_dispatch: + schedule: + # run every week (for --pre release tests) + - cron: "0 0 * * 0" + +# cancel in-progress runs that use the same workflow and branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-manifest: + # check-manifest is a tool that checks that all files in version control are + # included in the sdist (unless explicitly excluded) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: pipx run check-manifest + + test: + name: ${{ matrix.platform }} (${{ matrix.python-version }}) + runs-on: ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", ] # "3.12"] + platform: [ubuntu-latest, ] # ...macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + + - name: ๐ Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache-dependency-path: "pyproject.toml" + cache: "pip" + + - name: Install Dependencies + run: | + python -m pip install -U pip + # if running a cron job, we add the --pre flag to test against pre-releases + python -m pip install .[test] ${{ github.event_name == 'schedule' && '--pre' || '' }} + + - name: ๐งช Run Tests + run: pytest --color=yes --cov --cov-report=xml --cov-report=term-missing + + # If something goes wrong with --pre tests, we can open an issue in the repo + - name: ๐ Report --pre Failures + if: failure() && github.event_name == 'schedule' + uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PLATFORM: ${{ matrix.platform }} + PYTHON: ${{ matrix.python-version }} + RUN_ID: ${{ github.run_id }} + TITLE: "[test-bot] pip install --pre is failing" + with: + filename: .github/TEST_FAIL_TEMPLATE.md + update_existing: true + + - name: Coverage + uses: codecov/codecov-action@v3 + + deploy: + name: Deploy + needs: test + if: success() && startsWith(github.ref, 'refs/tags/') && github.event_name != 'schedule' + runs-on: ubuntu-latest + + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing on PyPi + # see https://docs.pypi.org/trusted-publishers/ + id-token: write + # This permission allows writing releases + contents: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: ๐ Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: ๐ท Build + run: | + python -m pip install build + python -m build + + - name: ๐ข Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + - uses: softprops/action-gh-release@v1 + with: + generate_release_notes: true + files: './dist/*' diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml deleted file mode 100644 index 6616f06..0000000 --- a/.github/workflows/test_and_deploy.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: tests - -on: - push: - branches: - - main - tags: - - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 - pull_request: - branches: - - main - workflow_dispatch: - -jobs: - test: - name: ${{ matrix.platform }} py${{ matrix.python-version }} - runs-on: ${{ matrix.platform }} - strategy: - matrix: - platform: [ ubuntu-latest ] - python-version: ["3.9", "3.10" , "3.11"] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools pytest - pip install -e . - - - name: Test with pytest - run: pytest - env: - PLATFORM: ${{ matrix.platform }} - - - name: Coverage - uses: codecov/codecov-action@v1 - - deploy: - # this will run when you have tagged a commit, starting with "v*" - # and requires that you have put your twine API key in your - # github secrets (see readme for details) - needs: [ test ] - runs-on: ubuntu-latest - if: contains(github.ref, 'tags') - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U setuptools setuptools_scm wheel twine - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TWINE_API_KEY }} - run: | - git tag - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..49e3ffa --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +# enable pre-commit.ci at https://pre-commit.ci/ +# it adds: +# 1. auto fixing pull requests +# 2. auto updating the pre-commit configuration +ci: + autoupdate_schedule: monthly + autofix_commit_msg: "style(pre-commit.ci): auto fixes [...]" + autoupdate_commit_msg: "ci(pre-commit.ci): autoupdate" + +repos: + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.15 + hooks: + - id: validate-pyproject + + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.1.3 + hooks: + - id: ruff + args: [--fix] + + - repo: https://github.com/psf/black + rev: 23.10.1 + hooks: + - id: black + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.6.1 + hooks: + - id: mypy + files: "^src/" + # # you have to add the things you want to type check against here + # additional_dependencies: + # - numpy diff --git a/README.md b/README.md index 71f04c3..7330e29 100644 --- a/README.md +++ b/README.md @@ -1,142 +1,93 @@ # starfile -[![Build Status](https://travis-ci.com/alisterburt/starfile.svg?branch=master)](https://travis-ci.com/alisterburt/starfile) -[![PyPI version](https://badge.fury.io/py/starfile.svg)](https://pypi.python.org/pypi/starfile/) -[![PyPI pyversions](https://img.shields.io/pypi/pyversions/starfile.svg)](https://pypi.python.org/pypi/starfile/) -[![DOI](https://zenodo.org/badge/273026988.svg)](https://zenodo.org/badge/latestdoi/273026988) +[![License](https://img.shields.io/pypi/l/starfile.svg?color=green)](https://github.com/teamtomo/starfile/raw/main/LICENSE) +[![PyPI](https://img.shields.io/pypi/v/starfile.svg?color=green)](https://pypi.org/project/starfile) +[![Python Version](https://img.shields.io/pypi/pyversions/starfile.svg?color=green)](https://python.org) +[![CI](https://github.com/teamtomo/starfile/actions/workflows/ci.yml/badge.svg)](https://github.com/teamtomo/starfile/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/teamtomo/starfile/branch/main/graph/badge.svg)](https://codecov.io/gh/teamtomo/starfile) -`starfile` is a Python implementation of the [STAR](https://en.wikipedia.org/wiki/Self-defining_Text_Archive_and_Retrieval) -file format designed principally for compatibility with [RELION](https://github.com/3dem/relion) - format STAR files. +*starfile* is a package for reading and writing +[STAR files](https://en.wikipedia.org/wiki/Self-defining_Text_Archive_and_Retrieval) in Python. -It allows STAR files to be created and opened easily using a very simple API, exposing data blocks as [pandas](https://pandas.pydata.org/pandas-docs/stable/getting_started/overview.html) `DataFrame` objects. +
+ +
-This library aims to allow users and developers to read and write STAR files in Python as easily as possible as well as to encourage further analysis of data within the scientific Python ([SciPy](https://www.scipy.org/)) ecosystem. +*starfile* can be used interactively to inspect/explore files or in +scripts and larger software packages to provide basic STAR file I/O functions. +Data is exposed as simple python dictionaries or +[pandas dataframes](https://pandas.pydata.org/docs/user_guide/dsintro.html#dataframe). -You can use it interactively to inspect/explore files or in scripts and larger software packages to provide basic STAR file I/O functions. +This package was designed principally for compatibility with files generated by +[RELION](https://www3.mrc-lmb.cam.ac.uk/relion/index.php/Main_Page). -``` -The STAR file: a new format for electronic dataframes transfer and archiving -J. Chem. Inf. Comput. Sci. 1991, 31, 2, 326โ333 -Publication Date: May 1, 1991 -https://doi.org/10.1021/ci00002a020 -``` -## Features -- Easy to install and use -- Simple API for reading of STAR files as pandas `DataFrame` objects -- Simple API for writing of STAR files from pandas `DataFrame` objects +For more information on working with dataframes, please see the +[pandas docs](https://pandas.pydata.org/docs/user_guide/10min.html). +For *starfile* specific documentation, see [teamtomo.org/starfile](https://teamtomo.org/starfile) -## Installation -Installation is available directly from the [Python package index](https://pypi.org/project/starfile/) -```bash -pip install starfile -``` - -Currently `python` >= `3.8` is supported. You can check your `python` version with - -```sh -python -V -``` -We recommend installing into a [virtual environment](https://jni.github.io/using-python-for-science/intro-to-environments.html) for use in your projects. +--- +# Quickstart +For the following file `particles.star` with a single data block -## Usage +```txt +data_particles -### Reading STAR files -To open a STAR file -```python ->>> import starfile ->>> df = starfile.read('tests/dataframes/one_loop.star') ->>> df - rlnCoordinateX rlnCoordinateY ... rlnAngleTilt rlnAnglePsi -0 1572.444 1084.500 ... 0 0 -1 1507.500 1104.357 ... 0 0 -2 1512.432 973.500 ... 0 0 -3 1560.385 1063.500 ... 0 0 -4 1537.500 1060.500 ... 0 0 - ... ... ... ... ... -1360 1078.500 796.500 ... 0 0 -1361 1075.500 784.500 ... 0 0 -1362 1080.531 796.500 ... 0 0 -1363 1045.992 737.411 ... 0 0 -1364 1053.530 745.500 ... 0 0 - -[1365 rows x 12 columns] +loop_ +_rlnCoordinateX #1 +_rlnCoordinateY #2 +_rlnCoordinateZ #3 +_rlnAngleRot #4 +_rlnAngleTilt #5 +_rlnAnglePsi #6 +_rlnMicrographName #7 +91.798700 83.622600 203.341030 -51.740000 173.930000 32.971000 01_10.00Apx.mrc +97.635800 80.437000 203.136160 141.500000 171.760000 -134.680000 01_10.00Apx.mrc +92.415200 88.842700 210.663900 -78.750000 173.930000 87.263200 01_10.00Apx.mrc +94.607830 93.135410 205.425960 -85.215000 167.170000 85.632200 01_10.00Apx.mrc +86.187800 80.125400 204.558750 14.910000 163.260000 -16.030000 01_10.00Apx.mrc +91.824240 76.738300 203.794280 39.740000 168.410000 -57.250000 01_10.00Apx.mrc +98.253300 73.530100 203.856030 73.950000 166.380000 -84.640000 01_10.00Apx.mrc +101.303500 80.290800 194.790400 -178.878000 166.090000 73.181000 01_10.00Apx.mrc ``` -- Each block will return either a DataFrame for a `loop_` block, or a `dict` for a simple block -- A STAR file with a single block simply returns that DataFrame or `dict`, while a STAR file containing multiple blocks will return a `dict` of entries, with keys determined by the `data_*` titles -- If you would like to always return a `dict` of entries even for single-block STAR files, you can use the `always_dict=True` keyword argument - - -### Writing STAR files -DataFrame objects (or dicts or lists of dataframes) can be written to STAR files using `starfile.write` +Read the file ```python ->>> starfile.write(df, 'tests/dataframes/cars.star') -``` - -Produces a STAR file which looks like -```bash -# Created by the starfile python package (version 0.1) on 18/06/2020 13:26:32 -data_cars +import starfile -loop_ -_Brand #1 -_Price #2 -Honda_Civic 22000 -Toyota_Corolla 25000 -Ford_Focus 27000 -Audi_A4 35000 +df = starfile.read('particles.star') ``` +Interact with the data -- floating point format can be specified by the `float_format` keyword argument (default `%.6f`) -- data block headers are of format `data_