diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd96561b2..ee5167df7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,8 +20,9 @@ jobs: - cp3*-manylinux_x86_64 # MacOS wheels - cp3*-macosx_x86_64 - # Until we have arm64 runners, we can't automatically test arm64 wheels - cp3*-macosx_arm64 + # Windows wheels + - cp3*-win_amd64 sdist: true test_command: python -c "from stcal.ramp_fitting.ols_cas22 import _ramp, _jump, _fit; from stcal.ramp_fitting import slope_fitter" secrets: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5256d0fba..873d80ff1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,8 @@ jobs: - linux: py313-cov-xdist coverage: codecov - macos: py313-xdist + - windows: py313-xdist + python-version: '>=3.13.5 <3.14' test_downstream: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: diff --git a/setup.py b/setup.py index d5e47f0f4..5cd07512e 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,6 @@ import numpy as np +import os +import sys from Cython.Build import cythonize from Cython.Compiler import Options from setuptools import Extension, setup @@ -15,9 +17,22 @@ include_dirs = [np.get_include()] # Setup C module macros -define_macros = [("NUMPY", "1")] +define_macros = [ + ("NUMPY", "1"), +] + +# Setup C libraries +libraries = [] + +if sys.platform.startswith("win"): + define_macros.append(("PSAPI_VERSION", "1")) + libraries.append("psapi") + +debug_logdir= os.environ.get("DEBUG_LOGDIR") +if debug_logdir: + define_macros.append(("DEBUG_LOGDIR", f"\"{debug_logdir}\"")) -# importing these extension modules is tested in `.github/workflows/build.yml`; +# importing these extension modules is tested in `.github/workflows/build.yml`; # when adding new modules here, make sure to add them to the `test_command` entry there extensions = [ Extension( @@ -43,6 +58,7 @@ ["src/stcal/ramp_fitting/src/slope_fitter.c"], include_dirs=include_dirs, define_macros=define_macros, + libraries=libraries, ), ] diff --git a/src/stcal/outlier_detection/median.py b/src/stcal/outlier_detection/median.py index 2ff899fe1..75b8d1455 100644 --- a/src/stcal/outlier_detection/median.py +++ b/src/stcal/outlier_detection/median.py @@ -5,6 +5,7 @@ import logging import tempfile import warnings +import sys from pathlib import Path from typing import Any @@ -184,8 +185,13 @@ def __init__(self, msg = f"Invalid slice shape {slice_shape}. Only 2-D arrays are supported." raise ValueError(msg) self._filename = Path(filename) - with Path.open(self._filename, "wb") as f: # noqa: F841 - pass + try: + with Path.open(self._filename, "wb") as f: # noqa: F841 + pass + except PermissionError: + if self._filename.is_dir() and sys.platform.startswith("win"): + raise IsADirectoryError(self._filename) + self._slice_shape = slice_shape self._dtype = np.dtype(dtype) self._append_count = 0 diff --git a/src/stcal/ramp_fitting/src/slope_fitter.c b/src/stcal/ramp_fitting/src/slope_fitter.c index 845d98284..6e8415f34 100644 --- a/src/stcal/ramp_fitting/src/slope_fitter.c +++ b/src/stcal/ramp_fitting/src/slope_fitter.c @@ -7,9 +7,27 @@ #include #include #include -#include #include -#include + + +#if defined(_WIN32) // Windows +# include +# include +# include + +# ifndef PATH_MAX +# define PATH_MAX MAX_PATH +# endif + +# ifdef _WIN64 +# define ssize_t __int64 +# else +# define ssize_t long +# endif +#else // Linux/Unix +# include +# include +#endif #include #include @@ -98,11 +116,13 @@ const real_t LARGE_VARIANCE_THRESHOLD = 1.e6; * data structure itself. */ #define FREE_RAMP_DATA(RD) \ - if (RD) { \ - clean_ramp_data(rd); \ - free(RD); \ - (RD) = NULL; \ - } + do { \ + if (RD) { \ + clean_ramp_data(rd); \ + free(RD); \ + (RD) = NULL; \ + } \ + } while (0) /* * Wraps the clean_pixel_ramp function. Ensure all allocated @@ -111,10 +131,12 @@ const real_t LARGE_VARIANCE_THRESHOLD = 1.e6; * data structure itself. */ #define FREE_PIXEL_RAMP(PR) \ - if (PR) { \ - clean_pixel_ramp(PR); \ - SET_FREE(PR); \ - } + do { \ + if (PR) { \ + clean_pixel_ramp(PR); \ + SET_FREE(PR); \ + } \ + } while (0) /* * Wraps the clean_segment_list function. Ensure all allocated @@ -123,10 +145,12 @@ const real_t LARGE_VARIANCE_THRESHOLD = 1.e6; * data structure itself. */ #define FREE_SEGS_LIST(N, S) \ - if (S) { \ - clean_segment_list(N, S); \ - SET_FREE(S);\ - } + do { \ + if (S) { \ + clean_segment_list(N, S); \ + SET_FREE(S);\ + } \ + } while (0) /* Complicated dereferencing and casting using a label. */ #define VOID_2_FLOAT(A) (*((float*)(A))) @@ -798,7 +822,7 @@ print_delim_char(char c, int len) { printf("\n"); } -/* +/* * Used to determine if a pixel is in a list. * This is a debugging function. */ @@ -807,9 +831,8 @@ is_pix_in_list(struct ramp_data * rd, struct pixel_ramp * pr) { /* Pixel list */ // JP-3669 - (1804, 173) - const int len = 1; - npy_intp rows[len]; - npy_intp cols[len]; + npy_intp rows[1]; + npy_intp cols[1]; npy_intp row; int k; @@ -818,7 +841,7 @@ is_pix_in_list(struct ramp_data * rd, struct pixel_ramp * pr) rows[0] = 1804; cols[0] = 173; - for (k=0; krow + rd->start_row; if (row==rows[k] && pr->col==cols[k]) { return 1; @@ -827,29 +850,70 @@ is_pix_in_list(struct ramp_data * rd, struct pixel_ramp * pr) return 0; } -static inline long long +#define DEBUG_LOG_TS_FILE 0 +#define DEBUG_LOG_TS_MSG 1 +static inline void +get_log_timestamp(char *tbuffer, size_t len, int style) { + time_t now = time(NULL); + struct tm * curr_tm = localtime(&now); + const char * string_fmt; + switch (style) { + case 0: + string_fmt = "%Y_%m_%d_%H%M%S"; + break; + default: + string_fmt = "%Y/%m/%d %H:%M:%S"; + break; + } + + memset(tbuffer, 0, len); + strftime(tbuffer, len - 1, string_fmt, curr_tm); + return; +} + +static long long print_pid_info(long long prev, int line, char * label) { - struct rusage res_usage; - long long now_time = (long long)time(NULL); + char tbuffer[128]; long long mem_usage = -1; long long diff = 0; pid_t pid = getpid(); - // dbg_ols_print("PID: %d\n", pid); +#if defined(_WIN32) + HANDLE proc; + PROCESS_MEMORY_COUNTERS_EX pmc; + + proc = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + if (proc == NULL) { + return -1; + } - getrusage(RUSAGE_SELF, &res_usage); - mem_usage = res_usage.ru_maxrss; + if (GetProcessMemoryInfo(proc, (PROCESS_MEMORY_COUNTERS *) &pmc, sizeof(pmc))) { + mem_usage = pmc.WorkingSetSize; + } else { + return -1; + } + CloseHandle(proc); +#else + struct rusage res_usage; + if (getrusage(RUSAGE_SELF, &res_usage) < 0) { + perror("getrusage"); + return -1; + } + mem_usage = res_usage.ru_maxrss; +#endif + get_log_timestamp(tbuffer, sizeof(tbuffer), DEBUG_LOG_TS_MSG); + // dbg_ols_print("PID: %d\n", pid); if (prev > 0) { diff = mem_usage - prev; - dbg_ols_print("[%d] time: %lld, Mem: %lld, diff: %lld, prev: %lld, pid: %d '%s'\n", - line, now_time, mem_usage, diff, prev, pid, label); + dbg_ols_print("[%d] time: %s, Mem: %lld, diff: %lld, prev: %lld, pid: %d '%s'\n", + line, tbuffer, mem_usage, diff, prev, pid, label); } else { - dbg_ols_print("[%d] time: %lld, Mem: %lld, diff: %lld, pid: %d '%s'\n", - line, now_time, mem_usage, diff, pid, label); + dbg_ols_print("[%d] time: %s, Mem: %lld, diff: %lld, pid: %d '%s'\n", + line, tbuffer, mem_usage, diff, pid, label); } return mem_usage; } - + /* ------------------------------------------------------------------------- */ /* PROC LOGGER */ @@ -873,18 +937,15 @@ print_pid_info(long long prev, int line, char * label) { */ void set_up_logger() { - const char * log_dir = NULL; - char tbuffer[128]; - time_t now = time(NULL); - struct tm * curr_tm = localtime(&now); int sz; - const char * string_fmt = "%Y_%m_%d_%H%M%S"; - - return; + char tbuffer[128]; + const char *log_dir = NULL; - memset(tbuffer, 0, 128); - strftime(tbuffer, 127, string_fmt, curr_tm); + if (!log_dir) { + return; + } + get_log_timestamp(tbuffer, sizeof(tbuffer), DEBUG_LOG_TS_FILE); sz = snprintf(g_log_name, PATH_MAX-1, "%s/%s_pid_%d_logger.txt", log_dir, tbuffer, g_pid); @@ -931,35 +992,35 @@ ols_slope_fitter( /* Allocate, fill, and validate ramp data */ rd = get_ramp_data(args); if (NULL == rd) { - goto ERROR; + goto ERROR_END; } /* Prepare output products */ if (create_rate_product(&rate_prod, rd) || create_rateint_product(&rateint_prod, rd)) { - goto ERROR; + goto ERROR_END; } /* Prepare the pixel ramp data structure */ pr = create_pixel_ramp(rd); if (NULL==pr) { - goto ERROR; + goto ERROR_END; } /* Fit ramps for each pixel */ if (ols_slope_fit_pixels(rd, pr, &rate_prod, &rateint_prod)) { - goto ERROR; + goto ERROR_END; } /* Package up results to be returned */ result = package_results(&rate_prod, &rateint_prod, rd); if ((NULL==result) || (Py_None==(PyObject*)result)) { - goto ERROR; + goto ERROR_END; } goto CLEANUP; -ERROR: +ERROR_END: Py_XDECREF(result); /* Clean up errors */ @@ -1423,10 +1484,10 @@ create_opt_res( struct opt_res_product * opt_res, /* The optional results product */ struct ramp_data * rd) /* The ramp fitting data */ { - const npy_intp nd = 4; - npy_intp dims[nd]; - const npy_intp pnd = 3; - npy_intp pdims[pnd]; + npy_intp dims[4]; + npy_intp pdims[3]; + const npy_intp nd = sizeof(dims) / sizeof(*dims); + const npy_intp pnd = sizeof(pdims) / sizeof(*pdims); const int fortran = 0; /* Want C order */ const char * msg = "Couldn't allocate memory for opt_res products."; @@ -1568,8 +1629,8 @@ create_rate_product( struct rate_product * rate, /* The rate product */ struct ramp_data * rd) /* The ramp fitting data */ { - const npy_intp nd = 2; - npy_intp dims[nd]; + npy_intp dims[2]; + const npy_intp nd = sizeof(dims) / sizeof(*dims); const int fortran = 0; const char * msg = "Couldn't allocate memory for rate products."; @@ -1626,8 +1687,8 @@ create_rateint_product( struct rateint_product * rateint, /* The rateints product */ struct ramp_data * rd) /* The ramp fitting data */ { - const npy_intp nd = 3; - npy_intp dims[nd]; + npy_intp dims[3]; + const npy_intp nd = sizeof(dims) / sizeof(*dims); const int fortran = 0; const char * msg = "Couldn't allocate memory for rateint products."; @@ -3496,8 +3557,8 @@ save_opt_res( { void * ptr = NULL; npy_intp integ, crnum, segnum, row, col, idx; - const int msg_size = 1024; - char msg[msg_size]; + char msg[1024]; + const int msg_size = sizeof(msg); struct simple_ll_node * current; struct simple_ll_node * next; struct cr_node * cr_current; @@ -4207,8 +4268,13 @@ print_npy_types() { printf("NPY_DOUBLE = %d\n",NPY_DOUBLE); printf("NPY_VOID = %d\n",NPY_VOID); - printf("NPY_NTYPES_LEGACY = %d\n",NPY_NTYPES_LEGACY); +#if defined(NPY_NTYPES_LEGACY) + printf("NPY_NTYPES_LEGACY [np2 compat] = %d\n",NPY_NTYPES_LEGACY); +#elif defined(NPY_NTYPES) + printf("NPY_NTYPES [np1 compat] = %d\n",NPY_NTYPES); +#endif printf("NPY_NOTYPE = %d\n",NPY_NOTYPE); + /* NPY_SHORT NPY_USHORT diff --git a/tests/outlier_detection/test_median.py b/tests/outlier_detection/test_median.py index 62a30dacc..fc0827513 100644 --- a/tests/outlier_detection/test_median.py +++ b/tests/outlier_detection/test_median.py @@ -24,7 +24,7 @@ def test_disk_appendable_array(tmp_path): arr = _DiskAppendableArray(slice_shape, dtype, fname) # check temporary file setup - assert str(arr._filename).split("/")[-1] in os.listdir(tempdir) # noqa: SLF001 + assert fname.exists() assert len(os.listdir(tempdir)) == 1 assert arr.shape == (0, *slice_shape)