From 6557773c8ddc73bbcae95ad58aa7d2caa11bd897 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Fri, 28 Jan 2022 09:30:34 +0100 Subject: [PATCH 01/98] Integrated code developed at ESS, so far --- pyscicat/client.py | 187 ++++++++++++++++++++++++++++++++++++++++++++- pyscicat/model.py | 12 +++ 2 files changed, 198 insertions(+), 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 9901055..6361882 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -10,7 +10,7 @@ import requests -from pyscicat.model import Attachment, Datablock, Dataset, RawDataset, DerivedDataset +from pyscicat.model import Attachment, Datablock, Dataset, OrigDatablock, RawDataset, DerivedDataset logger = logging.getLogger("splash_ingest") can_debug = logger.isEnabledFor(logging.DEBUG) @@ -67,6 +67,7 @@ def __init__( self._username = username # default username self._password = password # default password self._token = token # store token here + self._headers = {} # store headers assert self._base_url is not None, "SciCat database URL must be provided" logger.info(f"Starting ingestor talking to scicat at: {self._base_url}") @@ -76,6 +77,8 @@ def __init__( self._password is not None ), "SciCat login credentials (username, password) must be provided if token is not provided" self._token = get_token(self._base_url, self._username, self._password) + self._headers['Authorization'] = "Bearer {}".format(self._token) + def _send_to_scicat(self, url, dataDict=None, cmd="post"): """sends a command to the SciCat API server using url and token, returns the response JSON @@ -84,6 +87,7 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): response = requests.post( url, params={"access_token": self._token}, + headers=self._headers, json=dataDict, timeout=self._timeout_seconds, stream=False, @@ -93,6 +97,7 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): response = requests.delete( url, params={"access_token": self._token}, + headers=self._headers, timeout=self._timeout_seconds, stream=False, ) @@ -100,6 +105,7 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): response = requests.get( url, params={"access_token": self._token}, + headers=self._headers, json=dataDict, timeout=self._timeout_seconds, stream=False, @@ -108,6 +114,7 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): response = requests.patch( url, params={"access_token": self._token}, + headers=self._headers, json=dataDict, timeout=self._timeout_seconds, stream=False, @@ -170,6 +177,41 @@ def upload_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + + def upload_new_dataset(self, dataset: Dataset) -> str: + """ + Upload a new dataset. Uses the generic dataset endpoint. + Relys on the endpoint to sense wthe dataset type + + Parameters + ---------- + dataset : Dataset + Dataset to create + + Returns + ------- + dataset : Dataset + Dataset created including the pid (or unique identifier) of the newly created dataset + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + dataset_url = self._base_url + "Datasets" + resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) + + if not resp.ok: + err = resp.json()["error"] + raise ScicatCommError(f"Error creating dataset {err}") + + new_pid = resp.json().get("pid") + logger.info(f"new dataset created {new_pid}") + + return resp.json() + + + def upload_raw_dataset(self, dataset: Dataset) -> str: """Upload a raw dataset @@ -197,6 +239,8 @@ def upload_raw_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + + def upload_derived_dataset(self, dataset: Dataset) -> str: """Upload a derived dataset @@ -226,6 +270,8 @@ def upload_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + + def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): """Upload a Datablock @@ -234,6 +280,11 @@ def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets datablock : Datablock Datablock to upload + Returns + ------- + datablock : Datablock + The created Datablock with id + Raises ------ ScicatCommError @@ -249,6 +300,44 @@ def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets err = resp.json()["error"] raise ScicatCommError(f"Error creating datablock. {err}") + return resp.json() + + + + def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: + """ + Post SciCat Dataset OrigDatablock + + Parameters + ---------- + origdatablock : + The OrigDatablock to create + + Returns + ------- + dict + The created OrigDatablock with id + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + + """ + + encoded_pid = urllib.parse.quote_plus(origdatablock.datasetPid) + endpoint = "/Datasets/" + encoded_pid + "/origdatablocks" + url = self._base_url + endpoint + + resp = self._send_to_scicat(url, origdatablock.dict(exclude_none=True)) + if not resp.ok: + err = resp.json()["error"] + raise ScicatCommError(f"Error creating dataset original datablock. {err}") + + return resp.json() + + + def upload_attachment( self, attachment: Attachment, datasetType: str = "RawDatasets" ): @@ -320,6 +409,7 @@ def get_datasets_full_query(self, skip=0, limit=25, query_fields=None): return None return response.json() + def get_datasets(self, filter_fields=None) -> List[Dataset]: """Gets datasets using the simple fiter mechanism. This is appropriate when you do not require paging or text search, but @@ -352,6 +442,101 @@ def get_datasets(self, filter_fields=None) -> List[Dataset]: return response.json() + + def get_instrument(self, pid: str = None, name: str = None) -> dict: + """ + Get an instrument by pid or by name. + If pid is provided it takes priority over name. + + Parameters + ---------- + pid : str + Pid of the instrument + + name : str + The name of the instrument + + Returns + ------- + dict + The instrument with the requested name + """ + + if pid: + encoded_pid = parse.quote_plus(pid) + endpoint = "/Instruments/{}".format(encoded_pid) + url = self._base_url + endpoint + elif name: + endpoint = "/Instruments/findOne" + query = json.dumps({"where": {"name": {"like": name}}}) + url = self._base_url + endpoint + "?" + query + else: + logger.error("Invalid instrument pid and/or name. You must specify instrument pid or name") + return None + + response = self._send_to_scicat(url, cmd="get") + if not response.ok: + err = response.json()["error"] + logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') + return None + return response.json() + + + + def get_sample(self, pid: str = None) -> dict: + """ + Get an sample by pid. + + Parameters + ---------- + pid : str + The pid of the sample + + Returns + ------- + dict + The sample with the requested pid + """ + + encoded_pid = parse.quote_plus(pid) + endpoint = "/Samples/{}".format(encoded_pid) + url = self._base_url + endpoint + response = self._send_to_scicat(url, cmd="get") + if not response.ok: + err = response.json()["error"] + logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') + return None + return response.json() + + + + def get_proposal(self, pid: str = None) -> dict: + """ + Get proposal by pid. + + Parameters + ---------- + pid : str + The pid of the proposal + + Returns + ------- + dict + The proposal with the requested pid + """ + + endpoint = "/Proposals/" + url = self._base_url + endpoint + pid + response = self._send_to_scicat(url, cmd="get") + if not response.ok: + err = response.json()["error"] + logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') + return None + return response.json() + + + + def get_file_size(pathobj): filesize = pathobj.lstat().st_size return filesize diff --git a/pyscicat/model.py b/pyscicat/model.py index f9f26fc..6de34c4 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -198,6 +198,18 @@ class Datablock(Ownable): datasetId: str +class OrigDatablock(Ownable): + """ + A Original Datablock maps between a Dataset and contains DataFiles + """ + + id: Optional[str] + # archiveId: str = None listed in catamel model, but comes back invalid? + size: int + dataFileList: List[DataFile] + datasetId: str + + class Attachment(Ownable): """ Attachments can be any base64 encoded string...thumbnails are attachments From c9ae0a1d7b380737b33ff10853fc4f1e58a7ee34 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Fri, 28 Jan 2022 15:00:10 -0800 Subject: [PATCH 02/98] REL: v0.2.1 From 7d052747c92516eb9426c5ea9f576322bdcf3c32 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Fri, 28 Jan 2022 15:21:24 -0800 Subject: [PATCH 03/98] url fix --- pyscicat/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 9901055..74b2247 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -60,6 +60,8 @@ def __init__( timeout_seconds : [int], optional timeout in seconds to wait for http connections to return, by default None """ + if base_url[-1] != "/": + base_url = base_url + "/" self._base_url = base_url self._timeout_seconds = ( timeout_seconds # we are hitting a transmission timeout... @@ -392,7 +394,8 @@ def get_token(base_url, username, password): """logs in using the provided username / password combination and receives token for further communication use""" logger.info(f" Getting new token for user {username}") - + if base_url[-1] != "/": + base_url = base_url + "/" response = requests.post( base_url + "Users/login", json={"username": username, "password": password}, From 0b6f6dc9a913af5f5923cba123557ef7c856fbb6 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Fri, 28 Jan 2022 15:21:53 -0800 Subject: [PATCH 04/98] REL: v0.2.2 From c63e3e2e35332410282d17703dfb0240fffcd086 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Thu, 17 Feb 2022 11:51:08 +0100 Subject: [PATCH 05/98] Fixed linting issues --- pyscicat/client.py | 60 +++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 36 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 3863c40..627c480 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -10,7 +10,14 @@ import requests -from pyscicat.model import Attachment, Datablock, Dataset, OrigDatablock, RawDataset, DerivedDataset +from pyscicat.model import ( + Attachment, + Datablock, + Dataset, + OrigDatablock, + RawDataset, + DerivedDataset, +) logger = logging.getLogger("splash_ingest") can_debug = logger.isEnabledFor(logging.DEBUG) @@ -69,7 +76,7 @@ def __init__( self._username = username # default username self._password = password # default password self._token = token # store token here - self._headers = {} # store headers + self._headers = {} # store headers assert self._base_url is not None, "SciCat database URL must be provided" logger.info(f"Starting ingestor talking to scicat at: {self._base_url}") @@ -79,8 +86,7 @@ def __init__( self._password is not None ), "SciCat login credentials (username, password) must be provided if token is not provided" self._token = get_token(self._base_url, self._username, self._password) - self._headers['Authorization'] = "Bearer {}".format(self._token) - + self._headers["Authorization"] = "Bearer {}".format(self._token) def _send_to_scicat(self, url, dataDict=None, cmd="post"): """sends a command to the SciCat API server using url and token, returns the response JSON @@ -99,7 +105,7 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): response = requests.delete( url, params={"access_token": self._token}, - headers=self._headers, + headers=self._headers, timeout=self._timeout_seconds, stream=False, ) @@ -107,7 +113,7 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): response = requests.get( url, params={"access_token": self._token}, - headers=self._headers, + headers=self._headers, json=dataDict, timeout=self._timeout_seconds, stream=False, @@ -116,7 +122,7 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): response = requests.patch( url, params={"access_token": self._token}, - headers=self._headers, + headers=self._headers, json=dataDict, timeout=self._timeout_seconds, stream=False, @@ -179,7 +185,6 @@ def upload_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid - def upload_new_dataset(self, dataset: Dataset) -> str: """ Upload a new dataset. Uses the generic dataset endpoint. @@ -206,13 +211,11 @@ def upload_new_dataset(self, dataset: Dataset) -> str: if not resp.ok: err = resp.json()["error"] raise ScicatCommError(f"Error creating dataset {err}") - + new_pid = resp.json().get("pid") logger.info(f"new dataset created {new_pid}") - - return resp.json() - + return resp.json() def upload_raw_dataset(self, dataset: Dataset) -> str: """Upload a raw dataset @@ -241,8 +244,6 @@ def upload_raw_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid - - def upload_derived_dataset(self, dataset: Dataset) -> str: """Upload a derived dataset @@ -272,8 +273,6 @@ def upload_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid - - def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): """Upload a Datablock @@ -304,15 +303,13 @@ def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets return resp.json() - - def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: """ Post SciCat Dataset OrigDatablock Parameters ---------- - origdatablock : + origdatablock : The OrigDatablock to create Returns @@ -338,8 +335,6 @@ def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: return resp.json() - - def upload_attachment( self, attachment: Attachment, datasetType: str = "RawDatasets" ): @@ -411,7 +406,6 @@ def get_datasets_full_query(self, skip=0, limit=25, query_fields=None): return None return response.json() - def get_datasets(self, filter_fields=None) -> List[Dataset]: """Gets datasets using the simple fiter mechanism. This is appropriate when you do not require paging or text search, but @@ -454,8 +448,6 @@ def get_datasets(self, filter_fields=None) -> List[Dataset]: # return None # return response.json() - - def get_instrument(self, pid: str = None, name: str = None) -> dict: """ Get an instrument by pid or by name. @@ -464,7 +456,7 @@ def get_instrument(self, pid: str = None, name: str = None) -> dict: Parameters ---------- pid : str - Pid of the instrument + Pid of the instrument name : str The name of the instrument @@ -475,8 +467,8 @@ def get_instrument(self, pid: str = None, name: str = None) -> dict: The instrument with the requested name """ - if pid: - encoded_pid = parse.quote_plus(pid) + if pid: + encoded_pid = urllib.parse.quote_plus(pid) endpoint = "/Instruments/{}".format(encoded_pid) url = self._base_url + endpoint elif name: @@ -484,9 +476,11 @@ def get_instrument(self, pid: str = None, name: str = None) -> dict: query = json.dumps({"where": {"name": {"like": name}}}) url = self._base_url + endpoint + "?" + query else: - logger.error("Invalid instrument pid and/or name. You must specify instrument pid or name") + logger.error( + "Invalid instrument pid and/or name. You must specify instrument pid or name" + ) return None - + response = self._send_to_scicat(url, cmd="get") if not response.ok: err = response.json()["error"] @@ -494,8 +488,6 @@ def get_instrument(self, pid: str = None, name: str = None) -> dict: return None return response.json() - - def get_sample(self, pid: str = None) -> dict: """ Get an sample by pid. @@ -511,7 +503,7 @@ def get_sample(self, pid: str = None) -> dict: The sample with the requested pid """ - encoded_pid = parse.quote_plus(pid) + encoded_pid = urllib.parse.quote_plus(pid) endpoint = "/Samples/{}".format(encoded_pid) url = self._base_url + endpoint response = self._send_to_scicat(url, cmd="get") @@ -521,8 +513,6 @@ def get_sample(self, pid: str = None) -> dict: return None return response.json() - - def get_proposal(self, pid: str = None) -> dict: """ Get proposal by pid. @@ -548,8 +538,6 @@ def get_proposal(self, pid: str = None) -> dict: return response.json() - - def get_file_size(pathobj): filesize = pathobj.lstat().st_size return filesize From 90b64f01540d015f44810929869bdc5efacf3362 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Thu, 24 Feb 2022 16:51:57 +0100 Subject: [PATCH 06/98] Fixed issue in update_dataset_origdatablock --- pyscicat/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 627c480..3056850 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -324,8 +324,8 @@ def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: """ - encoded_pid = urllib.parse.quote_plus(origdatablock.datasetPid) - endpoint = "/Datasets/" + encoded_pid + "/origdatablocks" + encoded_pid = urllib.parse.quote_plus(origdatablock.datasetId) + endpoint = "Datasets/" + encoded_pid + "/origdatablocks" url = self._base_url + endpoint resp = self._send_to_scicat(url, origdatablock.dict(exclude_none=True)) From df29d4c1df46a3cbda3759231df82dbba079ebfa Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Tue, 8 Mar 2022 11:11:21 +0100 Subject: [PATCH 07/98] added documentation and example --- .../howto/ingestion_simulation_dataset_ess.md | 245 +++++++++++++++++ ...gestion_simulation_dataset_ess_config.json | 7 + ...estion_simulation_dataset_ess_dataset.json | 253 ++++++++++++++++++ examples/ingestion_sumulation_dataset_ess.py | 86 ++++++ 4 files changed, 591 insertions(+) create mode 100644 docs/source/howto/ingestion_simulation_dataset_ess.md create mode 100644 examples/data/ingestion_simulation_dataset_ess_config.json create mode 100644 examples/data/ingestion_simulation_dataset_ess_dataset.json create mode 100644 examples/ingestion_sumulation_dataset_ess.py diff --git a/docs/source/howto/ingestion_simulation_dataset_ess.md b/docs/source/howto/ingestion_simulation_dataset_ess.md new file mode 100644 index 0000000..b715092 --- /dev/null +++ b/docs/source/howto/ingestion_simulation_dataset_ess.md @@ -0,0 +1,245 @@ +# Ingest Simulation Dataset at ESS +In the process of designing and commissioning of the ESS, many simulation datasets have been produced in the process of finding the best design and validate them. +At ESS, we have decided to import such datsets in to our SciCat instance to facilitate search, assess quickly the comulative quality of the collected results and be able to start applying Machine Learning techniques to such data in the near future. + +## Background +Data scientist and modeller at ESS have produced many simulations each one including multiple variations of the same design running parameters exploration. +The process of ingesting all this information into SciCat will produce around a thousands new datasets. +To facilitate testing and validation of all the information at each step of the process, data curators have decided to break down the process in multiple scripts which comulative collect all the information needed to create a meaningful entry in SciCat. +The process produces one json file containing the basic information, metadata and files associated with one datasets. +The last step is to read such file and inges it into SciCat. +The rest of this document covers all the code used to load the dataset information, create the matching models and create a new dataset and orig datablock in SciCat. + +## Individual Dataset entry +Each dataset is prepared for ingestion and save in an individual json file. +The example json file is available under the example/data folder and has the following structure: + +```json +{ + "id": "0275d813-be6b-444f-812f-b8311d129361", + "dataset": { + "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", + "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", + "principalInvestigator": "Max Novelli", + "creationLocation": "DMSC", + "owner": "Massimiliano Novelli", + "ownerEmail": "max.novelli@ess.eu", + "contactEmail": "max.novelli@ess.eu", + "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", + "creationTime": "2022-03-07T15:44:59.000Z", + "type": "raw", + "techniques": [ + { + "pid": "fe888574-5cc0-11ec-90c3-bf82943dec35", + "name": "Simulation" + } + ], + "size": 68386784, + "instrumentId": "", + "sampleId": "", + "proposalId": "", + "scientificMetadata": { + "sample_width": { + "value": 0.015, + "unit": "m" + }, + "sample_height": { + "value": 0.015, + "unit": "m" + }, + "divergence_requirement_horizontal": { + "value": 0.75, + "unit": "deg" + }, + ... omissed ... + } + }, + "orig_datablock": { + "size": 68386784, + "ownerGroup": "ess", + "accessGroups": ["dmsc", "swap"], + "dataFileList": [ + { + "path": "launch_all.sh", + "size": 10171, + "time": "2014-01-23T19:52:37.000Z" + }, { + "path": "suggested_reruns-fails.sh", + "size": 448, + "time": "2014-01-23T19:53:04.000Z" + }, + ... omissed ... + ] + }, + "ownable": { + "ownerGroup": "ess", + "accessGroups": ["dmsc"] + } +} + +``` +As you can see, the file has already been structure with the three main component of the dataset: +- the main dataset body with scientifica metadata +- the ownable object +- the orig datablock containing all the files tassociated with the dataset + +The three sections allows for an easier ingestion code + +## Script +The script to ingest the dataset mentioned above is available in the exampe folder with the name of `ingestion_simulation_dataset_ess.py`. +In this section, we are going to walk through the code of this script to illustrate the various functionalities. + + +### Overall decription +The ingestion is organized in simple sections by leveraging the dataset information which is already optimally optimized to peerform the operations required to create a full dataset in SciCat. +In order to simplify the script, it is assumed that pyscicat is installed system wide and the script is run from the folder where is saved. All the file paths are relative to the script folder. +At the beginning of the script, libraries are imported and we define paths to the relevant json files. + +```python +# libraries +import json +import pyscicat.client as pyScClient +import pyscicat.model as pyScModel + + +# scicat configuration file +# includes scicat instance URL +# scicat user and password +scicat_configuration_file = "./data/ingestion_simulation_dataset_ess_config.json" +simulation_dataset_file = "./data/ingestion_simulation_dataset_ess_dataset.json" +``` + + +### Loading relevant information +In the next section, the script loads the configuration needed to communicate with SciCat and the dataset information + +```python +# loads scicat configuration +with open(scicat_configuration_file,"r") as fh: + scicat_config = json.load(fh) + + +# loads simulation information from matching json file +with open(simulation_dataset_file,"r") as fh: + dataset_information = json.load(fh) +``` + + +### Authentication +Here, we instantiate the pyscicat object and perform the login. + +```python +scClient = pyScClient.ScicatClient( + base_url=scicat_config['scicat']['host'], + username=scicat_config['scicat']['username'], + password=scicat_config['scicat']['password'] +) +``` + + +### Create Ownable model +We, than, instantiate the ownable object, which is used in assign the correct owner and access to all the other SciCat entries that we are going to create. + +```python +ownable = pyScModel.Ownable( + **dataset_information['ownable'] +) +``` + +This notiation is equivalent to pass in all the ownable object properties explicitly. +```python +ownable = pyScModel.Ownable( + ownerGroup=dataset_information['ownable']['ownergroup'], + accessGroups=dataset_information['ownable']['accessGroups'] +) +``` + + +### Create Dataset model +Next step, we need to instantiate a raw dataset object defined in pySciCat models. +Make sure to select the correct dataset: raw or derived. In our case, we are creating a raw one, which is specified in the dataset json file +```python +dataset = pyScModel.RawDataset( + **dataset_information['dataset'], + **ownable.dict() +) +``` + +As highlighted in the previous section, this notation is equivalent to assign all the model properties explicitly: +```python +dataset = pyScModel.RawDataset( + datasetName=dataset_information['dataset']['datasetName'], + description=dataset_information['dataset']['description'], + creationLocation=dataset_information['dataset']['creationLocation'], + principalInvestigator=dataset_information['dataset']['principalInvestigator'], + owner=dataset_information['dataset']['owner'], + ownerEmail=dataset_information['dataset']['ownerEmail'], + ... omitted ... + ownerGroup=dataset_information['ownable']['ownergroup'], + accessGroups=dataset_information['ownable']['accessGroups'] +) +``` + + +### Submit Dataset to SciCat +We are now ready to make a post to SciCat and create a Dataset + +```python +created_dataset = scClient.upload_new_dataset(dataset) +``` + +If the request is successful, the variable created_dataset should return the same information present in dataset with the additionl field named _pid_ which cotnains the official pid assigned to this dataset by SciCat + + +### Create OrigDatablock model +Now that we have created the dataset, we will add the list of files related to this dataset. +As we have done with the other objects, we leverage the pySciCat model to make sure that the information is properly validated. +In this snippet of code, we use explicit notation for the main object, and we use the expansion for the inner file model. + +```python +origDataBlock = pyScModel.OrigDatablock( + size=dataset_information['orig_datablock']['size'], + datasetId=created_dataset['pid'], + dataFileList=[ + pyScModel.DataFile( + **file + ) + for file + in dataset_information['orig_datablock']['dataFileList'] + ], + **ownable.dict() +) +``` + +As highlighted before, this code is equivalent to: +```python +origDataBlock = pyScModel.OrigDatablock( + size=dataset_information['orig_datablock']['size'], + datasetId=created_dataset['pid'], + dataFileList=[ + pyScModel.DataFile( + path=file['path',] + size=file['size'], + time=file['time'] + ) + for file + in dataset_information['orig_datablock']['dataFileList'] + ], + ownerGroup=dataset_information['ownable']['ownergroup'], + accessGroups=dataset_information['ownable']['accessGroups'] +) +``` + +### Submit OrigDatablock +With the original datablock object created, it is time to submit th erequest to SciCat. + +```python +created_orig_datablock = scClient.upload_dataset_origdatablock(origDataBlock) +``` + +Similarly to the dataset creation function, this call will return the same information provided as argument, with the addition of the pid assigned to the entry by SciCat + + +## Validate the dataset +At this point, you can visit your instance of SciCat and you should see the dataset that we just created in the list of datasets. The file list can be viewed visiting the _Datafiles_ tab on the dataset details page + diff --git a/examples/data/ingestion_simulation_dataset_ess_config.json b/examples/data/ingestion_simulation_dataset_ess_config.json new file mode 100644 index 0000000..6b49bdd --- /dev/null +++ b/examples/data/ingestion_simulation_dataset_ess_config.json @@ -0,0 +1,7 @@ +{ + "scicat" : { + "host": "", + "username": "ingestor", + "password": "" + } +} diff --git a/examples/data/ingestion_simulation_dataset_ess_dataset.json b/examples/data/ingestion_simulation_dataset_ess_dataset.json new file mode 100644 index 0000000..2c90117 --- /dev/null +++ b/examples/data/ingestion_simulation_dataset_ess_dataset.json @@ -0,0 +1,253 @@ +{ + "id": "0275d813-be6b-444f-812f-b8311d129361", + "dataset": { + "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", + "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", + "creationLocation": "DMSC", + "principalInvestigator": "Max Novelli", + "scientificMetadata": { + "sample_width": { "value": 0.015, "unit": "m" }, + "sample_height": { "value": 0.015, "unit": "m" }, + "divergence_requirement_horizontal": { "value": 0.75, "unit": "deg" }, + "divergence_requirement_vertical": { "value": 1, "unit": "deg" }, + "guide_sample_distance": { "value": 0.6, "unit": "m" }, + "lower_wavelength_limit": { "value": 1, "unit": "\u00c5" }, + "upper_wavelength_limit": { "value": 3.6, "unit": "\u00c5" }, + "moderator_width": { "value": 0.12, "unit": "m" }, + "moderator_height": { "value": 0.03, "unit": "m" }, + "moderator_sample_distance": { "value": 170, "unit": "m" }, + "parsing_variables": { "value": "guide_start , startx1 , starty1 , length1", "unit": "" }, + "parsing_min_guide_start": { "value": 2.000035881054106, "unit": "m" }, + "parsing_max_guide_start": { "value": 5.407538318585075, "unit": "m" }, + "parsing_mean_guide_start": { "value": 2.3475508029429557, "unit": "m" }, + "parsing_std_guide_start": { "value": 0.5522363822422368, "unit": "m" }, + "parsing_min_startx1": { "value": 0.006706596967962139, "unit": "m" }, + "parsing_max_startx1": { "value": 0.1460959338571846, "unit": "m" }, + "parsing_mean_startx1": { "value": 0.08885675463366878, "unit": "m" }, + "parsing_std_startx1": { "value": 0.017699812942929365, "unit": "m" }, + "parsing_min_starty1": { "value": 0.011762187831963904, "unit": "m" }, + "parsing_max_starty1": { "value": 0.14999127413576652, "unit": "m" }, + "parsing_mean_starty1": { "value": 0.13009670276273638, "unit": "m" }, + "parsing_std_starty1": { "value": 0.011522927034872269, "unit": "m" }, + "parsing_min_length1": { "value": 28.915197821153896, "unit": "" }, + "parsing_max_length1": { "value": 95.07944574028325, "unit": "" }, + "parsing_mean_length1": { "value": 64.23126877070395, "unit": "" }, + "parsing_std_length1": { "value": 10.210341803833671, "unit": "" }, + "optimization_name": { "value": "PGESKSE", "unit": "" }, + "configuration_summary": { "value": "PGESKSE", "unit": "" }, + "best_figure_of_merit": { "value": "0.25293", "unit": "" }, + "brilliance_transfer": { "value": "0.47344", "unit": "" }, + "event_file_name_suffix": { "value": "4Hsize_3moderator_size_y", "unit": "" }, + "number_of_parameters": { "value": 2, "unit": "" }, + "parameters_name": { "value": "Hsize , moderator_size_y", "unit": "" }, + "event_writen_present": { "value": true, "unit": "" }, + "event_writen_file": { "value": "master_record-writen_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_writen_timestamp": { "value": "2014-01-23T19:52:38", "unit": "" }, + "event_done_present": { "value": true, "unit": "" }, + "event_done_file": { "value": "master_record-done_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_done_timestamp": { "value": "2014-01-25T00:35:55", "unit": "" }, + "event_analysis_present": { "value": true, "unit": "" }, + "event_analysis_file": { "value": "output/analysis/master_record-analyzed_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_analysis_timestamp": { "value": "2014-01-28T14:03:02", "unit": "" }, + "dataset_name": { "value": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", "unit": "" }, + "run_name": { "value": "CAMEA CAMEA31", "unit": "" }, + "scan_name": { "value": "4Hsize_3moderator_size_y", "unit": "" }, + "output_file_name_base": { "value": "PGESKSE_4Hsize_3moderator_size_y", "unit": "" }, + "dataset_access_path": { "value": "/mnt/data/simulation/CAMEA/CAMEA31", "unit": "" }, + "parameters_structure": { "value": "[{\"name\": \"Hsize\", \"value\": \"1.5\", \"index\": \"4\"}, {\"name\": \"moderator_size_y\", \"value\": \"0.03\", \"index\": \"3\"}]", "unit": "" }, + "Hsize": { "value": 4, "unit": "cm" }, + "moderator_size_y": { "value": 3, "unit": "m" } + }, + "owner": "Massimiliano Novelli", + "ownerEmail": "max.novelli@ess.eu", + "contactEmail": "max.novelli@ess.eu", + "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", + "creationTime": "2022-03-07T15:44:59.000Z", + "type": "raw", + "techniques": [ + { + "pid": "fe888574-5cc0-11ec-90c3-bf82943dec35", + "name": "Simulation" + } + ], + "size": 68386784, + "instrumentId": "", + "sampleId": "", + "proposalId": "" + }, + "orig_datablock": { + "size": 68386784, + "dataFileList": [ + { + "path": "launch_all.sh", + "size": 10171, + "time": "2014-01-23T19:52:37.000Z" + }, + { + "path": "suggested_reruns-fails.sh", + "size": 448, + "time": "2014-01-23T19:53:04.000Z" + }, + { + "path": "compile_all_py.sh", + "size": 273, + "time": "2014-01-23T19:52:37.000Z" + }, + { + "path": "clean3.sh", + "size": 354, + "time": "2014-01-25T10:44:54.000Z" + }, + { + "path": "master_record-done_4Hsize_3moderator_size_y.txt", + "size": 579, + "time": "2014-01-25T00:35:55.000Z" + }, + { + "path": "master_record-writen_4Hsize_3moderator_size_y.txt", + "size": 561, + "time": "2014-01-23T19:52:38.000Z" + }, + { + "path": "compile_all.sh", + "size": 259, + "time": "2014-01-23T19:52:37.000Z" + }, + { + "path": "output/brill_ref/brilliance_ref_4Hsize_3moderator_size_y.mat", + "size": 11624010, + "time": "2014-01-24T07:56:45.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_acceptance_ess.png", + "size": 521132, + "time": "2014-01-27T11:38:06.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_acceptance_pure.png", + "size": 518423, + "time": "2014-01-27T11:37:52.000Z" + }, + { + "path": "output/analysis/master_record-analyzed_4Hsize_3moderator_size_y.txt", + "size": 587, + "time": "2014-01-28T14:03:02.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_overall_pure.png", + "size": 144605, + "time": "2014-01-27T11:37:49.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_posdiv_ess.png", + "size": 336496, + "time": "2014-01-27T11:38:04.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y_all.mat", + "size": 34321077, + "time": "2014-01-25T00:35:55.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_overall_ess.png", + "size": 127660, + "time": "2014-01-27T11:38:02.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_geometry.dat", + "size": 2175, + "time": "2014-01-25T00:23:10.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y_ifit_analyse.m", + "size": 19482, + "time": "2014-01-23T19:52:40.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_geometry.png", + "size": 76259, + "time": "2014-01-27T11:38:09.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_posdiv_pure.png", + "size": 353828, + "time": "2014-01-27T11:37:50.000Z" + }, + { + "path": "brilliance_refference/brilliance_ifit_4Hsize_3moderator_size_y.m", + "size": 3048, + "time": "2014-01-23T19:52:33.000Z" + }, + { + "path": "brilliance_refference/brilliance_4Hsize_3moderator_size_y1.mat", + "size": 11626979, + "time": "2014-01-24T07:56:42.000Z" + }, + { + "path": "brilliance_refference/brilliance_4Hsize_3moderator_size_y.batch", + "size": 671, + "time": "2014-01-23T19:52:32.000Z" + }, + { + "path": "brilliance_refference/input_used_4Hsize_3moderator_size_y.txt", + "size": 358, + "time": "2014-01-23T19:52:35.000Z" + }, + { + "path": "brilliance_refference/run_brilliance_ifit_4Hsize_3moderator_size_y.m", + "size": 53, + "time": "2014-01-23T19:52:36.000Z" + }, + { + "path": "PGESKSE/PGESKSE_4Hsize_3moderator_size_y.batch", + "size": 734, + "time": "2014-01-23T19:52:48.000Z" + }, + { + "path": "PGESKSE/PGESKSE_4Hsize_3moderator_size_y_ifit.m", + "size": 11101, + "time": "2014-01-23T19:52:48.000Z" + }, + { + "path": "PGESKSE/err_PGESKSE_4Hsize_3moderator_size_y.txt", + "size": 0, + "time": "2014-01-24T21:13:29.000Z" + }, + { + "path": "PGESKSE/run_PGESKSE_4Hsize_3moderator_size_y_ifit.m", + "size": 50, + "time": "2014-01-23T19:52:51.000Z" + }, + { + "path": "PGESKSE/out_PGESKSE_4Hsize_3moderator_size_y.txt", + "size": 8681220, + "time": "2014-01-25T00:35:58.000Z" + }, + { + "path": "PGESKSE/compile_PGESKSE_py.sh", + "size": 558, + "time": "2014-01-23T19:52:45.000Z" + }, + { + "path": "PGESKSE/compile_PGESKSE.sh", + "size": 540, + "time": "2014-01-23T19:52:45.000Z" + }, + { + "path": "PGESKSE/PGESKSE_4Hsize_3moderator_size_y1.par", + "size": 918, + "time": "2014-01-25T00:35:55.000Z" + }, + { + "path": "PGESKSE/PGESKSE_4Hsize_3moderator_size_y1_geometry.dat", + "size": 2175, + "time": "2014-01-25T00:23:10.000Z" + } + ] + }, + "ownable": { + "ownerGroup": "ess", + "accessGroups": ["dmsc"] + } +} + \ No newline at end of file diff --git a/examples/ingestion_sumulation_dataset_ess.py b/examples/ingestion_sumulation_dataset_ess.py new file mode 100644 index 0000000..c431ba3 --- /dev/null +++ b/examples/ingestion_sumulation_dataset_ess.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# coding: utf-8 + +# ingestion_simulation_dataset_ess +# +# Ingest the example simulation dataset in the specified scicat instance +# This script is provided as is, and as an example in pyScicat documentation +# +# +# Create by: Max Novelli +# max.novelli@ess.eu +# European Spallation Source ERIC, +# P.O. Box 176, +# SE-221 00, Lund, Sweden +# +# + + +# libraries +import json +import pyscicat.client as pyScClient +import pyscicat.model as pyScModel + + +# scicat configuration file +# includes scicat instance URL +# scicat user and password +scicat_configuration_file = "./data/ingestion_simulation_dataset_ess_config.json" +simulation_dataset_file = "./data/ingestion_simulation_dataset_ess.json" + + +# loads scicat configuration +with open(scicat_configuration_file,"r") as fh: + scicat_config = json.load(fh) + + +# loads simulation information from matching json file +with open(simulation_dataset_file,"r") as fh: + dataset_information = json.load(fh) + +# instantiate a pySciCat client +scClient = pyScClient.ScicatClient( + base_url=scicat_config['scicat']['host'], + username=scicat_config['scicat']['username'], + password=scicat_config['scicat']['password'] +) + +# create an owneable object to be used with all the other models +# all the fields are retrieved directly from the simulation information +ownable = pyScModel.Ownable( + **dataset_information['ownable'] +) + + +# create dataset object from the pyscicat model +# includes ownable from previous step +dataset = pyScModel.RawDataset( + **dataset_information['dataset'], + **ownable.dict() +) + + +# create dataset entry in scicat +# it returns the full dataset information, including the dataset pid assigned automatically by scicat +created_dataset = scClient.upload_new_dataset(dataset) + + +# create origdatablock object from pyscicat model +origDataBlock = pyScModel.OrigDatablock( + size=dataset_information['orig_datablock']['size'], + datasetId=created_dataset['pid'], + dataFileList=[ + pyScModel.DataFile( + **file + ) + for file + in dataset_information['orig_datablock']['dataFileList'] + ], + **ownable.dict() +) + +# create origDatablock associated with dataset in SciCat +# it returns the full object including SciCat id assigned when created +created_orig_datablock = scClient.upload_dataset_origdatablock(origDataBlock) + + From fd40529aff67194101cc3000f0ab4bbcce0046c1 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Wed, 9 Mar 2022 10:47:21 +0100 Subject: [PATCH 08/98] fixed linting --- examples/ingestion_sumulation_dataset_ess.py | 42 ++++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/examples/ingestion_sumulation_dataset_ess.py b/examples/ingestion_sumulation_dataset_ess.py index c431ba3..96c57c6 100644 --- a/examples/ingestion_sumulation_dataset_ess.py +++ b/examples/ingestion_sumulation_dataset_ess.py @@ -5,14 +5,14 @@ # # Ingest the example simulation dataset in the specified scicat instance # This script is provided as is, and as an example in pyScicat documentation -# +# # # Create by: Max Novelli # max.novelli@ess.eu -# European Spallation Source ERIC, -# P.O. Box 176, +# European Spallation Source ERIC, +# P.O. Box 176, # SE-221 00, Lund, Sweden -# +# # @@ -30,34 +30,29 @@ # loads scicat configuration -with open(scicat_configuration_file,"r") as fh: +with open(scicat_configuration_file, "r") as fh: scicat_config = json.load(fh) # loads simulation information from matching json file -with open(simulation_dataset_file,"r") as fh: +with open(simulation_dataset_file, "r") as fh: dataset_information = json.load(fh) # instantiate a pySciCat client scClient = pyScClient.ScicatClient( - base_url=scicat_config['scicat']['host'], - username=scicat_config['scicat']['username'], - password=scicat_config['scicat']['password'] + base_url=scicat_config["scicat"]["host"], + username=scicat_config["scicat"]["username"], + password=scicat_config["scicat"]["password"], ) # create an owneable object to be used with all the other models -# all the fields are retrieved directly from the simulation information -ownable = pyScModel.Ownable( - **dataset_information['ownable'] -) +# all the fields are retrieved directly from the simulation information +ownable = pyScModel.Ownable(**dataset_information["ownable"]) # create dataset object from the pyscicat model # includes ownable from previous step -dataset = pyScModel.RawDataset( - **dataset_information['dataset'], - **ownable.dict() -) +dataset = pyScModel.RawDataset(**dataset_information["dataset"], **ownable.dict()) # create dataset entry in scicat @@ -67,14 +62,11 @@ # create origdatablock object from pyscicat model origDataBlock = pyScModel.OrigDatablock( - size=dataset_information['orig_datablock']['size'], - datasetId=created_dataset['pid'], + size=dataset_information["orig_datablock"]["size"], + datasetId=created_dataset["pid"], dataFileList=[ - pyScModel.DataFile( - **file - ) - for file - in dataset_information['orig_datablock']['dataFileList'] + pyScModel.DataFile(**file) + for file in dataset_information["orig_datablock"]["dataFileList"] ], **ownable.dict() ) @@ -82,5 +74,3 @@ # create origDatablock associated with dataset in SciCat # it returns the full object including SciCat id assigned when created created_orig_datablock = scClient.upload_dataset_origdatablock(origDataBlock) - - From 56d2d34f51114528bfd3b44cd4ddfb5859476051 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Thu, 10 Mar 2022 13:29:35 +0100 Subject: [PATCH 09/98] updated documentation, included new documentation under how-to section of documentation, updated example. Added packages in requirements to requirements-dev. --- .../source/howto/ingestion_simulation_dataset_ess.md | 9 ++++++--- docs/source/index.md | 1 + .../ingestion_simulation_dataset_ess_dataset.json | 12 ++++++------ requirements-dev.txt | 3 +++ 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/source/howto/ingestion_simulation_dataset_ess.md b/docs/source/howto/ingestion_simulation_dataset_ess.md index b715092..5491b60 100644 --- a/docs/source/howto/ingestion_simulation_dataset_ess.md +++ b/docs/source/howto/ingestion_simulation_dataset_ess.md @@ -51,7 +51,9 @@ The example json file is available under the example/data folder and has the fol "value": 0.75, "unit": "deg" }, - ... omissed ... + "omissed" : { + "notes" : "Additional scientific metadata has been omitted for readability" + } } }, "orig_datablock": { @@ -67,8 +69,9 @@ The example json file is available under the example/data folder and has the fol "path": "suggested_reruns-fails.sh", "size": 448, "time": "2014-01-23T19:53:04.000Z" - }, - ... omissed ... + }, { + "notes" : "Additional files entries has been omitted for readability" + } ] }, "ownable": { diff --git a/docs/source/index.md b/docs/source/index.md index c32ff5a..1fae2e9 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -17,6 +17,7 @@ installation ```{toctree} :caption: How To Guides howto/ingest +howto/ingestion_simulation_dataset_ess ``` diff --git a/examples/data/ingestion_simulation_dataset_ess_dataset.json b/examples/data/ingestion_simulation_dataset_ess_dataset.json index 2c90117..ebd88bd 100644 --- a/examples/data/ingestion_simulation_dataset_ess_dataset.json +++ b/examples/data/ingestion_simulation_dataset_ess_dataset.json @@ -5,6 +5,12 @@ "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", "creationLocation": "DMSC", "principalInvestigator": "Max Novelli", + "owner": "Massimiliano Novelli", + "ownerEmail": "max.novelli@ess.eu", + "contactEmail": "max.novelli@ess.eu", + "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", + "creationTime": "2022-03-07T15:44:59.000Z", + "type": "raw", "scientificMetadata": { "sample_width": { "value": 0.015, "unit": "m" }, "sample_height": { "value": 0.015, "unit": "m" }, @@ -58,12 +64,6 @@ "Hsize": { "value": 4, "unit": "cm" }, "moderator_size_y": { "value": 3, "unit": "m" } }, - "owner": "Massimiliano Novelli", - "ownerEmail": "max.novelli@ess.eu", - "contactEmail": "max.novelli@ess.eu", - "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", - "creationTime": "2022-03-07T15:44:59.000Z", - "type": "raw", "techniques": [ { "pid": "fe888574-5cc0-11ec-90c3-bf82943dec35", diff --git a/requirements-dev.txt b/requirements-dev.txt index 2759b55..f3ccc33 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,6 @@ +# These are packages required to load pyscicat +pydantic +requests # These are required for developing the package (running the tests, building # the documentation) but not necessarily required for _using_ it. codecov From e7ebd0ee30fa953d9df136a2bb41b4d6207bb8b2 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Fri, 11 Mar 2022 07:28:26 +0100 Subject: [PATCH 10/98] removed production packages from requirements-dev --- requirements-dev.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f3ccc33..2759b55 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,3 @@ -# These are packages required to load pyscicat -pydantic -requests # These are required for developing the package (running the tests, building # the documentation) but not necessarily required for _using_ it. codecov From ab4616216e214fa1adbef7f522ea5b2dd280800f Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Mon, 14 Mar 2022 11:41:39 +0100 Subject: [PATCH 11/98] Renamed example script, added test for new functions --- ...py => ingestion_simulation_dataset_ess.py} | 0 pyscicat/tests/test_client.py | 4 + pyscicat/tests/test_new_dataset.py | 93 +++++++++++++++++++ 3 files changed, 97 insertions(+) rename examples/{ingestion_sumulation_dataset_ess.py => ingestion_simulation_dataset_ess.py} (100%) create mode 100644 pyscicat/tests/test_new_dataset.py diff --git a/examples/ingestion_sumulation_dataset_ess.py b/examples/ingestion_simulation_dataset_ess.py similarity index 100% rename from examples/ingestion_sumulation_dataset_ess.py rename to examples/ingestion_simulation_dataset_ess.py diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 5c470e6..69cf2f1 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -37,6 +37,9 @@ def add_mock_requests(mock_request): json={"response": "random"}, ) + mock_request.post(local_url + "Datasets", json={"pid": "17"}) + + def test_scicate_ingest(): with requests_mock.Mocker() as mock_request: @@ -105,3 +108,4 @@ def test_initializers(): client = from_token(local_url, "let me in!") assert client._token == "let me in!" + diff --git a/pyscicat/tests/test_new_dataset.py b/pyscicat/tests/test_new_dataset.py new file mode 100644 index 0000000..fabfa58 --- /dev/null +++ b/pyscicat/tests/test_new_dataset.py @@ -0,0 +1,93 @@ +from pathlib import Path +import urllib +import json + +import requests_mock +from ..client import ( + ScicatClient +) + +from ..model import ( + DataFile, + RawDataset, + OrigDatablock, + Ownable, +) + +global test_dataset + +local_url = "http://localhost:3000/api/v3/" +test_dataset_file = "../../examples/data/ingestion_simulation_dataset_ess_dataset.json" +test_dataset = None + +def set_up_test_environment(mock_request): + + global test_dataset + + # load test data + data_file_path = Path(__file__).parent.joinpath(test_dataset_file).resolve(); + with open(data_file_path,'r') as fh: + test_dataset = json.load(fh) + + mock_request.post( + local_url + "Users/login", + json={"id": "a_token"}, + ) + + mock_request.post( + local_url + "Datasets", + json={ + **{"pid": test_dataset['id']}, + **test_dataset['dataset'] + } + ) + + encoded_pid = urllib.parse.quote_plus(test_dataset['id']) + mock_request.post( + local_url + "Datasets/" + encoded_pid + "/origdatablocks", + json={ + "size" : test_dataset["orig_datablock"]["size"], + "datasetId" : test_dataset["id"], + "dataFileList" : test_dataset["orig_datablock"]["dataFileList"] + } + ) + + + +def test_scicate_ingest_raw_dataset(): + with requests_mock.Mocker() as mock_request: + set_up_test_environment(mock_request) + scicat = ScicatClient( + base_url=local_url, + username="Zaphod", + password="heartofgold", + ) + assert ( + scicat._token == "a_token" + ), "scicat client set the token given by the server" + + ownable = Ownable(ownerGroup="magrathea", accessGroups=["deep_though"]) + + # Create Dataset + dataset = RawDataset( + **test_dataset['dataset'], + **ownable.dict() + ) + created_dataset = scicat.upload_new_dataset(dataset) + + assert created_dataset['pid'] == test_dataset['id'] + + + # origDatablock with DataFiles + origDataBlock = OrigDatablock( + size=test_dataset["orig_datablock"]["size"], + datasetId=created_dataset["pid"], + dataFileList=[ + DataFile(**file) + for file in test_dataset["orig_datablock"]["dataFileList"] + ], + **ownable.dict() + ) + created_origdatablock = scicat.upload_dataset_origdatablock(origDataBlock) + assert len(created_origdatablock['dataFileList']) == len(test_dataset['orig_datablock']["dataFileList"]) + From d13e0539cfaaa0588dbc6df85ca8f9aa9483e96a Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Mon, 14 Mar 2022 11:45:58 +0100 Subject: [PATCH 12/98] Fixed linting --- pyscicat/tests/test_client.py | 2 -- pyscicat/tests/test_new_dataset.py | 40 ++++++++++++------------------ 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 69cf2f1..39ba2ca 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -40,7 +40,6 @@ def add_mock_requests(mock_request): mock_request.post(local_url + "Datasets", json={"pid": "17"}) - def test_scicate_ingest(): with requests_mock.Mocker() as mock_request: add_mock_requests(mock_request) @@ -108,4 +107,3 @@ def test_initializers(): client = from_token(local_url, "let me in!") assert client._token == "let me in!" - diff --git a/pyscicat/tests/test_new_dataset.py b/pyscicat/tests/test_new_dataset.py index fabfa58..e530db4 100644 --- a/pyscicat/tests/test_new_dataset.py +++ b/pyscicat/tests/test_new_dataset.py @@ -3,9 +3,7 @@ import json import requests_mock -from ..client import ( - ScicatClient -) +from ..client import ScicatClient from ..model import ( DataFile, @@ -20,13 +18,14 @@ test_dataset_file = "../../examples/data/ingestion_simulation_dataset_ess_dataset.json" test_dataset = None + def set_up_test_environment(mock_request): global test_dataset # load test data - data_file_path = Path(__file__).parent.joinpath(test_dataset_file).resolve(); - with open(data_file_path,'r') as fh: + data_file_path = Path(__file__).parent.joinpath(test_dataset_file).resolve() + with open(data_file_path, "r") as fh: test_dataset = json.load(fh) mock_request.post( @@ -35,25 +34,21 @@ def set_up_test_environment(mock_request): ) mock_request.post( - local_url + "Datasets", - json={ - **{"pid": test_dataset['id']}, - **test_dataset['dataset'] - } + local_url + "Datasets", + json={**{"pid": test_dataset["id"]}, **test_dataset["dataset"]}, ) - encoded_pid = urllib.parse.quote_plus(test_dataset['id']) + encoded_pid = urllib.parse.quote_plus(test_dataset["id"]) mock_request.post( local_url + "Datasets/" + encoded_pid + "/origdatablocks", json={ - "size" : test_dataset["orig_datablock"]["size"], - "datasetId" : test_dataset["id"], - "dataFileList" : test_dataset["orig_datablock"]["dataFileList"] - } + "size": test_dataset["orig_datablock"]["size"], + "datasetId": test_dataset["id"], + "dataFileList": test_dataset["orig_datablock"]["dataFileList"], + }, ) - def test_scicate_ingest_raw_dataset(): with requests_mock.Mocker() as mock_request: set_up_test_environment(mock_request) @@ -69,14 +64,10 @@ def test_scicate_ingest_raw_dataset(): ownable = Ownable(ownerGroup="magrathea", accessGroups=["deep_though"]) # Create Dataset - dataset = RawDataset( - **test_dataset['dataset'], - **ownable.dict() - ) + dataset = RawDataset(**test_dataset["dataset"], **ownable.dict()) created_dataset = scicat.upload_new_dataset(dataset) - assert created_dataset['pid'] == test_dataset['id'] - + assert created_dataset["pid"] == test_dataset["id"] # origDatablock with DataFiles origDataBlock = OrigDatablock( @@ -89,5 +80,6 @@ def test_scicate_ingest_raw_dataset(): **ownable.dict() ) created_origdatablock = scicat.upload_dataset_origdatablock(origDataBlock) - assert len(created_origdatablock['dataFileList']) == len(test_dataset['orig_datablock']["dataFileList"]) - + assert len(created_origdatablock["dataFileList"]) == len( + test_dataset["orig_datablock"]["dataFileList"] + ) From 834e9cedb3a0882354eeb6b8f3783e5fe81f21f8 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Mon, 28 Mar 2022 11:38:05 +0200 Subject: [PATCH 13/98] Added function to delete dataset by pid --- pyscicat/client.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pyscicat/client.py b/pyscicat/client.py index 3056850..8316a09 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -538,6 +538,33 @@ def get_proposal(self, pid: str = None) -> dict: return response.json() + def delete_dataset(self, pid: str = None) -> dict: + """ + Delete dataset by pid + + Parameters + ---------- + pid : str + The pid of the dataset to be deleted + + Returns + ------- + dict + response from SciCat backend + """ + + encoded_pid = urllib.parse.quote_plus(pid) + endpoint = "/Datasets/{}".format(encoded_pid) + url = self._base_url + endpoint + response = self._send_to_scicat(url, cmd='delete') + if not response.ok: + err = response.json()["error"] + logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') + return None + return response.json() + + + def get_file_size(pathobj): filesize = pathobj.lstat().st_size return filesize From 5cfed34df73ebb37e62f81f966e1004852ec1e36 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Wed, 13 Apr 2022 14:06:24 +0100 Subject: [PATCH 14/98] Added raw and derived upsert functions --- pyscicat/client.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/pyscicat/client.py b/pyscicat/client.py index 9901055..6d30521 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -226,6 +226,81 @@ def upload_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: + """Upsert a raw dataset + + Parameters + ---------- + dataset : Dataset + Dataset to load + + filter_fields + Filters to locate where to upsert dataset + + Returns + ------- + str + pid (or unique identifier) of the dataset + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + query_results = self.get_datasets(filter_fields) + if query_results.json(): + filter_fields = json.dumps(filter_fields) + raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?{{"where":{filter_fields}}}' + resp = self._send_to_scicat(raw_dataset_url, dataset.dict(exclude_none=True)) + if not resp.ok: + err = resp.json()["error"] + raise ScicatCommError(f"Error upserting raw dataset {err}") + new_pid = resp.json().get("pid") + logger.info(f"dataset updated {new_pid}") + return new_pid + else: + logger.info(f"dataset does not exist, could not upsert") + raise ScicatCommError(f"Dataset does not exist, could not upsert.") + + def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: + """Upsert a derived dataset + + Parameters + ---------- + dataset : Dataset + Dataset to upsert + + filter_fields + Filters to locate where to upsert dataset + + Returns + ------- + str + pid (or unique identifier) of the dataset + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + + query_results = self.get_datasets(filter_fields) + if query_results.json(): + filter_fields = json.dumps(filter_fields) + derived_dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?{{"where":{filter_fields}}}' + resp = self._send_to_scicat( + derived_dataset_url, dataset.dict(exclude_none=True) + ) + if not resp.ok: + err = resp.json()["error"] + raise ScicatCommError(f"Error upserting derived dataset {err}") + new_pid = resp.json().get("pid") + logger.info(f"dataset updated {new_pid}") + return new_pid + else: + logger.info(f"dataset does not exist, could not upsert") + raise ScicatCommError(f"Dataset does not exist, could not upsert.") + def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): """Upload a Datablock From 832b476f9921304e7054dc1fb8d70165d7545536 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Tue, 19 Apr 2022 15:48:54 +0200 Subject: [PATCH 15/98] added functions to load dataset by id and related orig_datablocks --- pyscicat/client.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 8316a09..c3b30a2 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -437,6 +437,24 @@ def get_datasets(self, filter_fields=None) -> List[Dataset]: return None return response.json() + def get_dataset_by_pid(self, pid=None) -> Dataset: + """Gets dataset with the pid provided. + + Parameters + ---------- + pid : string + pid of the dataset requested. + """ + + encode_pid = urllib.parse.quote_plus(pid) + url = f"{self._base_url}/Datasets/{encode_pid}" + response = self._send_to_scicat(url, cmd="get") + if not response.ok: + err = response.json()["error"] + logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') + return None + return response.json() + # this method is future, needs testing. # def update_dataset(self, pid, fields: Dict): # response = self._send_to_scicat( @@ -537,6 +555,29 @@ def get_proposal(self, pid: str = None) -> dict: return None return response.json() + def get_dataset_origdatablocks(self, pid: str = None) -> dict: + """ + Get dataset orig datablocks by dataset pid. + + Parameters + ---------- + pid : str + The pid of the dataset + + Returns + ------- + dict + The orig_datablocks of the dataset with the requested pid + """ + + encoded_pid = urllib.parse.quote_plus(pid) + url = f"{self._base_url}/Datasets/{encoded_pid}/origdatablocks" + response = self._send_to_scicat(url, cmd="get") + if not response.ok: + err = response.json()["error"] + logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') + return None + return response.json() def delete_dataset(self, pid: str = None) -> dict: """ @@ -556,7 +597,7 @@ def delete_dataset(self, pid: str = None) -> dict: encoded_pid = urllib.parse.quote_plus(pid) endpoint = "/Datasets/{}".format(encoded_pid) url = self._base_url + endpoint - response = self._send_to_scicat(url, cmd='delete') + response = self._send_to_scicat(url, cmd="delete") if not response.ok: err = response.json()["error"] logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') @@ -564,7 +605,6 @@ def delete_dataset(self, pid: str = None) -> dict: return response.json() - def get_file_size(pathobj): filesize = pathobj.lstat().st_size return filesize From 96155e15440d9e8e5b9428d2f0c53c97013e6583 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Thu, 21 Apr 2022 10:34:40 +0100 Subject: [PATCH 16/98] Added upsert_dataset. Fixed bug in upsert funcs --- pyscicat/client.py | 47 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 6d30521..4c9c6b4 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -226,6 +226,49 @@ def upload_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + def upsert_dataset(self, dataset: Dataset, filter_fields) -> str: + """Upsert a dataset + + Parameters + ---------- + dataset : Dataset + Dataset to load + + filter_fields + Filters to locate where to upsert dataset + + Returns + ------- + str + pid (or unique identifier) of the dataset + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + query_results = self.get_datasets(filter_fields) + if query_results: + filter_fields = json.dumps(filter_fields) + if isinstance(dataset, RawDataset): + dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?{{"where":{filter_fields}}}' + elif isinstance(dataset, DerivedDataset): + dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?{{"where":{filter_fields}}}' + else: + logging.error( + "Dataset type not recognized, not Raw or Derived type" + ) + resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) + if not resp.ok: + err = resp.json()["error"] + raise ScicatCommError(f"Error upserting dataset {err}") + new_pid = resp.json().get("pid") + logger.info(f"dataset updated {new_pid}") + return new_pid + else: + logger.info(f"dataset does not exist, could not upsert") + raise ScicatCommError(f"Dataset does not exist, could not upsert.") + def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: """Upsert a raw dataset @@ -248,7 +291,7 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: Raises if a non-20x message is returned """ query_results = self.get_datasets(filter_fields) - if query_results.json(): + if query_results: filter_fields = json.dumps(filter_fields) raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?{{"where":{filter_fields}}}' resp = self._send_to_scicat(raw_dataset_url, dataset.dict(exclude_none=True)) @@ -285,7 +328,7 @@ def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: """ query_results = self.get_datasets(filter_fields) - if query_results.json(): + if query_results: filter_fields = json.dumps(filter_fields) derived_dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?{{"where":{filter_fields}}}' resp = self._send_to_scicat( From d7b23306b44ae54907725cd1454fac2a53af8ad9 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Thu, 21 Apr 2022 13:46:49 +0100 Subject: [PATCH 17/98] Removed unneeded f in log/error statements causing bug --- pyscicat/client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 4c9c6b4..00ba139 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -266,8 +266,8 @@ def upsert_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info(f"dataset updated {new_pid}") return new_pid else: - logger.info(f"dataset does not exist, could not upsert") - raise ScicatCommError(f"Dataset does not exist, could not upsert.") + logger.info("dataset does not exist, could not upsert") + raise ScicatCommError("Dataset does not exist, could not upsert.") def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: """Upsert a raw dataset @@ -302,8 +302,8 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info(f"dataset updated {new_pid}") return new_pid else: - logger.info(f"dataset does not exist, could not upsert") - raise ScicatCommError(f"Dataset does not exist, could not upsert.") + logger.info("dataset does not exist, could not upsert") + raise ScicatCommError("Dataset does not exist, could not upsert.") def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: """Upsert a derived dataset @@ -341,8 +341,8 @@ def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info(f"dataset updated {new_pid}") return new_pid else: - logger.info(f"dataset does not exist, could not upsert") - raise ScicatCommError(f"Dataset does not exist, could not upsert.") + logger.info("dataset does not exist, could not upsert") + raise ScicatCommError("Dataset does not exist, could not upsert.") def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): """Upload a Datablock From 094ce4d29d133550152aa86a9bc0936d2167f931 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Tue, 26 Apr 2022 13:15:22 +0100 Subject: [PATCH 18/98] Added initial upsert test --- pyscicat/tests/test_client.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 5c470e6..887dff6 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -15,6 +15,7 @@ Datablock, DataFile, Dataset, + RawDataset, Ownable, ) @@ -28,6 +29,10 @@ def add_mock_requests(mock_request): ) mock_request.post(local_url + "Samples", json={"sampleId": "dataset_id"}) mock_request.post(local_url + "RawDatasets/replaceOrCreate", json={"pid": "42"}) + mock_request.get(local_url + "Datasets?filter=%7B%22where%22%3A+%7B%22sampleId%22%3A+%22gargleblaster%22%7D%7D", + json = {"response": "random"}) + mock_request.post(local_url + "/RawDatasets/upsertWithWhere?where=%22%3A+%7B%22sampleId%22%3A+%22gargleblaster%22%7D%7D", + json={"pid": "42"}) mock_request.post( local_url + "RawDatasets/42/origdatablocks", json={"response": "random"}, @@ -59,7 +64,7 @@ def test_scicate_ingest(): assert size is not None # RawDataset - dataset = Dataset( + dataset = RawDataset( path="/foo/bar", size=42, owner="slartibartfast", @@ -78,6 +83,28 @@ def test_scicate_ingest(): ) dataset_id = scicat.upload_raw_dataset(dataset) + # new dataset + dataset = Dataset( + path="/foo/bar", + size=42, + owner="slartibartfast", + contactEmail="slartibartfast@magrathea.org", + creationLocation="magrathea", + creationTime=str(datetime.now()), + type="raw", + instrumentId="earth", + proposalId="deepthought", + dataFormat="planet", + principalInvestigator="A. Mouse", + sourceFolder="/foo/bar", + scientificMetadata={"a": "newfield"}, + sampleId="gargleblaster", + **ownable.dict() + ) + + dataset_id = scicat.upsert_raw_dataset(dataset, {"sampleId": "gargleblaster"}) + assert dataset_id.pid == "42" + # Datablock with DataFiles data_file = DataFile(path="/foo/bar", size=42) data_block = Datablock( From e0c27ecfc21e8b2bcc9bc88d4a63410ae7cf9c5c Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Tue, 26 Apr 2022 16:42:33 +0200 Subject: [PATCH 19/98] Added model and method for PublishedData --- pyscicat/client.py | 30 ++++++++++++++++++++++++++++++ pyscicat/model.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/pyscicat/client.py b/pyscicat/client.py index c3b30a2..1b8b923 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -17,6 +17,7 @@ OrigDatablock, RawDataset, DerivedDataset, + PublishedData, ) logger = logging.getLogger("splash_ingest") @@ -437,6 +438,35 @@ def get_datasets(self, filter_fields=None) -> List[Dataset]: return None return response.json() + def get_published_data(self, filter=None) -> List[PublishedData]: + """Gets published data using the simple fiter mechanism. This + is appropriate when you do not require paging or text search, but + want to be able to limit results based on items in the Dataset object. + + For example, a search for published data of a given doi would have + ```python + filter = {"doi": "1234"} + ``` + + Parameters + ---------- + filter : dict + Dictionary of filtering fields. Must be json serializable. + """ + if not filter: + filter = None + else: + filter = json.dumps(filter) + + url = f'{self._base_url}/PublishedData' + f'?filter={{"where":{filter}}}' if filter else '' + response = self._send_to_scicat(url, cmd="get") + if not response.ok: + err = response.json()["error"] + logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') + return None + return response.json() + + def get_dataset_by_pid(self, pid=None) -> Dataset: """Gets dataset with the pid provided. diff --git a/pyscicat/model.py b/pyscicat/model.py index 6de34c4..d354f36 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -219,3 +219,31 @@ class Attachment(Ownable): thumbnail: str caption: Optional[str] datasetId: str + + +class PublishedData(): + """ + Published Data with registered DOI + """ + + doi: str + affiliation: str + creator: List[str] + publisher: str + publicationYear: int + title: str + url: Optional[str] + abstract: str + dataDescription: str + resourceType: str + numberOfFiles: Optional[int] + sizeOfArchive: Optional[int] + pidArray: List[str] + authors: List[str] + registeredTime: str + status: str + thumbnail: Optional[str] + createdBy: str + updatedBy: str + createdAt: str + updatedAt: str From 85877f36ee95cb36532b97f043b1c08c3c4e9a52 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Tue, 26 Apr 2022 16:48:47 +0200 Subject: [PATCH 20/98] Fixing bugs in published data functions --- pyscicat/client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 1b8b923..89397f1 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -458,7 +458,11 @@ def get_published_data(self, filter=None) -> List[PublishedData]: else: filter = json.dumps(filter) - url = f'{self._base_url}/PublishedData' + f'?filter={{"where":{filter}}}' if filter else '' + url = f'{self._base_url}/PublishedData' + ( + f'?filter={{"where":{filter}}}' + if filter + else '' + ) response = self._send_to_scicat(url, cmd="get") if not response.ok: err = response.json()["error"] From 5f0205cad9213995425816b929857ddaf61f7faf Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Wed, 27 Apr 2022 11:01:50 +0100 Subject: [PATCH 21/98] Fixed urls for upsert func and test --- pyscicat/client.py | 8 ++++---- pyscicat/tests/test_client.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 00ba139..145b86c 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -251,9 +251,9 @@ def upsert_dataset(self, dataset: Dataset, filter_fields) -> str: if query_results: filter_fields = json.dumps(filter_fields) if isinstance(dataset, RawDataset): - dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?{{"where":{filter_fields}}}' + dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' elif isinstance(dataset, DerivedDataset): - dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?{{"where":{filter_fields}}}' + dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' else: logging.error( "Dataset type not recognized, not Raw or Derived type" @@ -293,7 +293,7 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: query_results = self.get_datasets(filter_fields) if query_results: filter_fields = json.dumps(filter_fields) - raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?{{"where":{filter_fields}}}' + raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' resp = self._send_to_scicat(raw_dataset_url, dataset.dict(exclude_none=True)) if not resp.ok: err = resp.json()["error"] @@ -330,7 +330,7 @@ def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: query_results = self.get_datasets(filter_fields) if query_results: filter_fields = json.dumps(filter_fields) - derived_dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?{{"where":{filter_fields}}}' + derived_dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' resp = self._send_to_scicat( derived_dataset_url, dataset.dict(exclude_none=True) ) diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 887dff6..2080f05 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -29,9 +29,9 @@ def add_mock_requests(mock_request): ) mock_request.post(local_url + "Samples", json={"sampleId": "dataset_id"}) mock_request.post(local_url + "RawDatasets/replaceOrCreate", json={"pid": "42"}) - mock_request.get(local_url + "Datasets?filter=%7B%22where%22%3A+%7B%22sampleId%22%3A+%22gargleblaster%22%7D%7D", + mock_request.get(local_url + "/Datasets/?filter=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", json = {"response": "random"}) - mock_request.post(local_url + "/RawDatasets/upsertWithWhere?where=%22%3A+%7B%22sampleId%22%3A+%22gargleblaster%22%7D%7D", + mock_request.post(local_url + "/RawDatasets/upsertWithWhere?where=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", json={"pid": "42"}) mock_request.post( local_url + "RawDatasets/42/origdatablocks", @@ -103,7 +103,7 @@ def test_scicate_ingest(): ) dataset_id = scicat.upsert_raw_dataset(dataset, {"sampleId": "gargleblaster"}) - assert dataset_id.pid == "42" + assert dataset_id == "42" # Datablock with DataFiles data_file = DataFile(path="/foo/bar", size=42) From 9d31fe304dcf308f03144b25a57c44f3048412dd Mon Sep 17 00:00:00 2001 From: Dylan McReynolds <40469975+dylanmcreynolds@users.noreply.github.com> Date: Wed, 27 Apr 2022 11:24:21 -0700 Subject: [PATCH 22/98] remove pre-commit and re-add flake8 --- .github/workflows/linting.yml | 14 -------------- .github/workflows/testing.yml | 8 ++++++++ 2 files changed, 8 insertions(+), 14 deletions(-) delete mode 100644 .github/workflows/linting.yml diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml deleted file mode 100644 index 392e281..0000000 --- a/.github/workflows/linting.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.3 \ No newline at end of file diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5b95152..f3c75db 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -7,6 +7,7 @@ on: - cron: '00 4 * * *' # daily at 4AM jobs: + build: runs-on: ubuntu-latest @@ -33,6 +34,13 @@ jobs: set -vxeuo pipefail python -m pip install -r requirements-dev.txt python -m pip list + + - name: Lint with flake8 + shell: bash -l {0} + run: | + set -vxeuo pipefail + python -m flake8 + - name: Test with pytest shell: bash -l {0} run: | From 46737e1f8b490bd8f39fbe9e746021eebed68872 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds <40469975+dylanmcreynolds@users.noreply.github.com> Date: Wed, 27 Apr 2022 11:27:17 -0700 Subject: [PATCH 23/98] More remove pre-commit files --- .pre-commit-config.yaml | 15 --------------- requirements-dev.txt | 2 -- 2 files changed, 17 deletions(-) delete mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 0baa145..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -default_language_version: - python: python3 -repos: - - repo: https://github.com/ambv/black - rev: 21.12b0 - hooks: - - id: black - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.0.0 - hooks: - - id: flake8 - - repo: https://github.com/kynan/nbstripout - rev: 0.5.0 - hooks: - - id: nbstripout diff --git a/requirements-dev.txt b/requirements-dev.txt index 2759b55..7c4c01d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,9 +6,7 @@ flake8 pytest sphinx twine -pre-commit black -nbstripout requests_mock # These are dependencies of various sphinx extensions for documentation. ipython From 982793c5256f5e24eff69c55cfce10c48e5d450c Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Wed, 27 Apr 2022 11:37:47 -0700 Subject: [PATCH 24/98] remove pre-commit --- .github/workflows/linting.yml | 14 -------------- .github/workflows/testing.yml | 7 +++++++ .pre-commit-config.yaml | 15 --------------- requirements-dev.txt | 1 - 4 files changed, 7 insertions(+), 30 deletions(-) delete mode 100644 .github/workflows/linting.yml delete mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml deleted file mode 100644 index 392e281..0000000 --- a/.github/workflows/linting.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.3 \ No newline at end of file diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 5b95152..470a0e7 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -33,6 +33,13 @@ jobs: set -vxeuo pipefail python -m pip install -r requirements-dev.txt python -m pip list + + - name: Lint with flake8 + shell: bash -l {0} + run: | + set -vxeuo pipefail + python -m flake8 + - name: Test with pytest shell: bash -l {0} run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 0baa145..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -default_language_version: - python: python3 -repos: - - repo: https://github.com/ambv/black - rev: 21.12b0 - hooks: - - id: black - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.0.0 - hooks: - - id: flake8 - - repo: https://github.com/kynan/nbstripout - rev: 0.5.0 - hooks: - - id: nbstripout diff --git a/requirements-dev.txt b/requirements-dev.txt index 2759b55..42d5247 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,6 @@ flake8 pytest sphinx twine -pre-commit black nbstripout requests_mock From e4cdf4d3bae9abd2a4398813d965fd595792c1cd Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Thu, 28 Apr 2022 11:05:24 +0100 Subject: [PATCH 25/98] Black formatting fixes --- pyscicat/client.py | 12 ++++++------ pyscicat/tests/test_client.py | 14 ++++++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 145b86c..ea93b36 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -255,9 +255,7 @@ def upsert_dataset(self, dataset: Dataset, filter_fields) -> str: elif isinstance(dataset, DerivedDataset): dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' else: - logging.error( - "Dataset type not recognized, not Raw or Derived type" - ) + logging.error("Dataset type not recognized, not Raw or Derived type") resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) if not resp.ok: err = resp.json()["error"] @@ -294,7 +292,9 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: if query_results: filter_fields = json.dumps(filter_fields) raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - resp = self._send_to_scicat(raw_dataset_url, dataset.dict(exclude_none=True)) + resp = self._send_to_scicat( + raw_dataset_url, dataset.dict(exclude_none=True) + ) if not resp.ok: err = resp.json()["error"] raise ScicatCommError(f"Error upserting raw dataset {err}") @@ -330,9 +330,9 @@ def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: query_results = self.get_datasets(filter_fields) if query_results: filter_fields = json.dumps(filter_fields) - derived_dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' + dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' resp = self._send_to_scicat( - derived_dataset_url, dataset.dict(exclude_none=True) + dataset_url, dataset.dict(exclude_none=True) ) if not resp.ok: err = resp.json()["error"] diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 2080f05..fb2d256 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -29,10 +29,16 @@ def add_mock_requests(mock_request): ) mock_request.post(local_url + "Samples", json={"sampleId": "dataset_id"}) mock_request.post(local_url + "RawDatasets/replaceOrCreate", json={"pid": "42"}) - mock_request.get(local_url + "/Datasets/?filter=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", - json = {"response": "random"}) - mock_request.post(local_url + "/RawDatasets/upsertWithWhere?where=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", - json={"pid": "42"}) + mock_request.get( + local_url + + "/Datasets/?filter=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", + json={"response": "random"}, + ) + mock_request.post( + local_url + + "/RawDatasets/upsertWithWhere?where=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", + json={"pid": "42"}, + ) mock_request.post( local_url + "RawDatasets/42/origdatablocks", json={"response": "random"}, From 88ce96e3972b9bfb8d91dca3aba2d1f377a37768 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Thu, 28 Apr 2022 11:15:27 +0100 Subject: [PATCH 26/98] Raise ValueError in upsert_dataset --- pyscicat/client.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index ea93b36..0639160 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -247,15 +247,15 @@ def upsert_dataset(self, dataset: Dataset, filter_fields) -> str: ScicatCommError Raises if a non-20x message is returned """ + filters = json.dumps(filter_fields) + if isinstance(dataset, RawDataset): + dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filters}}}' + elif isinstance(dataset, DerivedDataset): + dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filters}}}' + else: + raise ValueError("Dataset type not recognised, not Raw or Derived type") query_results = self.get_datasets(filter_fields) if query_results: - filter_fields = json.dumps(filter_fields) - if isinstance(dataset, RawDataset): - dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - elif isinstance(dataset, DerivedDataset): - dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - else: - logging.error("Dataset type not recognized, not Raw or Derived type") resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) if not resp.ok: err = resp.json()["error"] @@ -331,9 +331,7 @@ def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: if query_results: filter_fields = json.dumps(filter_fields) dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - resp = self._send_to_scicat( - dataset_url, dataset.dict(exclude_none=True) - ) + resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) if not resp.ok: err = resp.json()["error"] raise ScicatCommError(f"Error upserting derived dataset {err}") From 6852dc5839ac9a1e483d19b01d5338c72e4e9d28 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Fri, 29 Apr 2022 11:14:37 +0100 Subject: [PATCH 27/98] Added additional upsert test case --- pyscicat/tests/test_client.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index fb2d256..4d97625 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -34,11 +34,21 @@ def add_mock_requests(mock_request): + "/Datasets/?filter=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", json={"response": "random"}, ) + mock_request.get( + local_url + + "/Datasets/?filter=%7B%22where%22:%7B%22sampleId%22:%20%22wowza%22%7D%7D", + json={"response": "random"}, + ) mock_request.post( local_url + "/RawDatasets/upsertWithWhere?where=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", json={"pid": "42"}, ) + mock_request.post( + local_url + + "/RawDatasets/upsertWithWhere?where=%7B%22where%22:%7B%22sampleId%22:%20%22wowza%22%7D%7D", + json={"pid": "54"}, + ) mock_request.post( local_url + "RawDatasets/42/origdatablocks", json={"response": "random"}, @@ -70,7 +80,7 @@ def test_scicate_ingest(): assert size is not None # RawDataset - dataset = RawDataset( + dataset = Dataset( path="/foo/bar", size=42, owner="slartibartfast", @@ -90,7 +100,7 @@ def test_scicate_ingest(): dataset_id = scicat.upload_raw_dataset(dataset) # new dataset - dataset = Dataset( + dataset = RawDataset( path="/foo/bar", size=42, owner="slartibartfast", @@ -108,9 +118,14 @@ def test_scicate_ingest(): **ownable.dict() ) + # Update existing record dataset_id = scicat.upsert_raw_dataset(dataset, {"sampleId": "gargleblaster"}) assert dataset_id == "42" + # Upsert non-existing record + dataset_id_2 = scicat.upsert_raw_dataset(dataset, {"sampleId": "wowza"}) + assert dataset_id_2 == "54" + # Datablock with DataFiles data_file = DataFile(path="/foo/bar", size=42) data_block = Datablock( From 29eeb9fffb0f2ef9fe6ebd534ca9d3efc5159c0f Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Fri, 29 Apr 2022 11:15:17 +0100 Subject: [PATCH 28/98] Removed generic upsert, use raw/derived instead --- pyscicat/client.py | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 0639160..266ab39 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -226,47 +226,6 @@ def upload_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid - def upsert_dataset(self, dataset: Dataset, filter_fields) -> str: - """Upsert a dataset - - Parameters - ---------- - dataset : Dataset - Dataset to load - - filter_fields - Filters to locate where to upsert dataset - - Returns - ------- - str - pid (or unique identifier) of the dataset - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - filters = json.dumps(filter_fields) - if isinstance(dataset, RawDataset): - dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filters}}}' - elif isinstance(dataset, DerivedDataset): - dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filters}}}' - else: - raise ValueError("Dataset type not recognised, not Raw or Derived type") - query_results = self.get_datasets(filter_fields) - if query_results: - resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error upserting dataset {err}") - new_pid = resp.json().get("pid") - logger.info(f"dataset updated {new_pid}") - return new_pid - else: - logger.info("dataset does not exist, could not upsert") - raise ScicatCommError("Dataset does not exist, could not upsert.") - def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: """Upsert a raw dataset From 66caeaa9e4152e9384b8c8e6adae95269c55c9b3 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Fri, 29 Apr 2022 11:31:01 +0100 Subject: [PATCH 29/98] Upsert inserts if dataset doesnt exist yet --- pyscicat/client.py | 53 ++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 266ab39..81aa357 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -248,21 +248,20 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: Raises if a non-20x message is returned """ query_results = self.get_datasets(filter_fields) - if query_results: - filter_fields = json.dumps(filter_fields) - raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - resp = self._send_to_scicat( - raw_dataset_url, dataset.dict(exclude_none=True) - ) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error upserting raw dataset {err}") - new_pid = resp.json().get("pid") - logger.info(f"dataset updated {new_pid}") - return new_pid - else: - logger.info("dataset does not exist, could not upsert") - raise ScicatCommError("Dataset does not exist, could not upsert.") + if not query_results: + logger.info("Dataset does not exist already, will be inserted") + filter_fields = json.dumps(filter_fields) + raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' + resp = self._send_to_scicat( + raw_dataset_url, dataset.dict(exclude_none=True) + ) + if not resp.ok: + err = resp.json()["error"] + raise ScicatCommError(f"Error upserting raw dataset {err}") + new_pid = resp.json().get("pid") + logger.info(f"dataset upserted {new_pid}") + return new_pid + def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: """Upsert a derived dataset @@ -287,19 +286,17 @@ def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: """ query_results = self.get_datasets(filter_fields) - if query_results: - filter_fields = json.dumps(filter_fields) - dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error upserting derived dataset {err}") - new_pid = resp.json().get("pid") - logger.info(f"dataset updated {new_pid}") - return new_pid - else: - logger.info("dataset does not exist, could not upsert") - raise ScicatCommError("Dataset does not exist, could not upsert.") + if not query_results: + logger.info("Dataset does not exist already, will be inserted") + filter_fields = json.dumps(filter_fields) + dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' + resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) + if not resp.ok: + err = resp.json()["error"] + raise ScicatCommError(f"Error upserting derived dataset {err}") + new_pid = resp.json().get("pid") + logger.info(f"dataset upserted {new_pid}") + return new_pid def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): """Upload a Datablock From 7b0465fc6ac20e936d9c88b0d4a4120f2ca3120b Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Mon, 9 May 2022 10:40:03 +0100 Subject: [PATCH 30/98] Black format fix --- pyscicat/client.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index b89b369..d90aea4 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -299,16 +299,13 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info("Dataset does not exist already, will be inserted") filter_fields = json.dumps(filter_fields) raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - resp = self._send_to_scicat( - raw_dataset_url, dataset.dict(exclude_none=True) - ) + resp = self._send_to_scicat(raw_dataset_url, dataset.dict(exclude_none=True)) if not resp.ok: err = resp.json()["error"] raise ScicatCommError(f"Error upserting raw dataset {err}") new_pid = resp.json().get("pid") logger.info(f"dataset upserted {new_pid}") return new_pid - def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: """Upsert a derived dataset From 1c1f1783f12100a433e74a38f90ad6f419964861 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Mon, 9 May 2022 15:37:59 +0100 Subject: [PATCH 31/98] Fixed indentation causing merge conflict --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f3c75db..fd2ce7e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -38,8 +38,8 @@ jobs: - name: Lint with flake8 shell: bash -l {0} run: | - set -vxeuo pipefail - python -m flake8 + set -vxeuo pipefail + python -m flake8 - name: Test with pytest shell: bash -l {0} From 3282dbcac974f6dd397a14a476d087db46bb6892 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Fri, 13 May 2022 10:11:04 +0100 Subject: [PATCH 32/98] Removed unneeded as_posix causing bug --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ecf6de..9e430c0 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def read_requirements_from_here(here: Path, filename: str = None) -> list: assert filename is not None, "filename as string must be provided" assert here.with_name( filename - ).exists(), f"requirements filename {filename.as_posix()} does not exist" + ).exists(), f"requirements filename {filename} does not exist" with open(here.with_name(filename)) as requirements_file: # Parse requirements.txt, ignoring any commented-out lines. requirements = [ From cb0fa2ca47afe11aca2716b6ff50304307d14dbe Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Fri, 3 Jun 2022 10:30:16 +0200 Subject: [PATCH 33/98] wip right before finalizing th ecode for the PR --- ...imulation_dataset_ess_derived_dataset.json | 257 ++++++++++++++++++ ...n_simulation_dataset_ess_raw_dataset.json} | 2 +- pyscicat/client.py | 223 +++++++++++++-- pyscicat/model.py | 2 +- pyscicat/tests/test_client.py | 2 +- pyscicat/tests/test_new_dataset.py | 116 ++++++-- 6 files changed, 562 insertions(+), 40 deletions(-) create mode 100644 examples/data/ingestion_simulation_dataset_ess_derived_dataset.json rename examples/data/{ingestion_simulation_dataset_ess_dataset.json => ingestion_simulation_dataset_ess_raw_dataset.json} (99%) diff --git a/examples/data/ingestion_simulation_dataset_ess_derived_dataset.json b/examples/data/ingestion_simulation_dataset_ess_derived_dataset.json new file mode 100644 index 0000000..1b0bcea --- /dev/null +++ b/examples/data/ingestion_simulation_dataset_ess_derived_dataset.json @@ -0,0 +1,257 @@ +{ + "id": "9be3bd96-e256-11ec-bd08-f32122965a87", + "dataset": { + "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE derived", + "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", + "investigator": "Max Novelli", + "inputDatasets" : ["0275d813-be6b-444f-812f-b8311d129361"], + "usedSoftware" : ["python","My software"], + "jobParameters" : { + "parameter-1" : "value-1", + "parameter-2" : "value-2" + }, + "jobLogData" : "Some jebrish about the dataset", + "owner": "Massimiliano Novelli", + "ownerEmail": "max.novelli@ess.eu", + "contactEmail": "max.novelli@ess.eu", + "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", + "creationTime": "2022-03-07T15:44:59.000Z", + "type": "derived", + "scientificMetadata": { + "sample_width": { "value": 0.015, "unit": "m" }, + "sample_height": { "value": 0.015, "unit": "m" }, + "divergence_requirement_horizontal": { "value": 0.75, "unit": "deg" }, + "divergence_requirement_vertical": { "value": 1, "unit": "deg" }, + "guide_sample_distance": { "value": 0.6, "unit": "m" }, + "lower_wavelength_limit": { "value": 1, "unit": "\u00c5" }, + "upper_wavelength_limit": { "value": 3.6, "unit": "\u00c5" }, + "moderator_width": { "value": 0.12, "unit": "m" }, + "moderator_height": { "value": 0.03, "unit": "m" }, + "moderator_sample_distance": { "value": 170, "unit": "m" }, + "parsing_variables": { "value": "guide_start , startx1 , starty1 , length1", "unit": "" }, + "parsing_min_guide_start": { "value": 2.000035881054106, "unit": "m" }, + "parsing_max_guide_start": { "value": 5.407538318585075, "unit": "m" }, + "parsing_mean_guide_start": { "value": 2.3475508029429557, "unit": "m" }, + "parsing_std_guide_start": { "value": 0.5522363822422368, "unit": "m" }, + "parsing_min_startx1": { "value": 0.006706596967962139, "unit": "m" }, + "parsing_max_startx1": { "value": 0.1460959338571846, "unit": "m" }, + "parsing_mean_startx1": { "value": 0.08885675463366878, "unit": "m" }, + "parsing_std_startx1": { "value": 0.017699812942929365, "unit": "m" }, + "parsing_min_starty1": { "value": 0.011762187831963904, "unit": "m" }, + "parsing_max_starty1": { "value": 0.14999127413576652, "unit": "m" }, + "parsing_mean_starty1": { "value": 0.13009670276273638, "unit": "m" }, + "parsing_std_starty1": { "value": 0.011522927034872269, "unit": "m" }, + "parsing_min_length1": { "value": 28.915197821153896, "unit": "" }, + "parsing_max_length1": { "value": 95.07944574028325, "unit": "" }, + "parsing_mean_length1": { "value": 64.23126877070395, "unit": "" }, + "parsing_std_length1": { "value": 10.210341803833671, "unit": "" }, + "optimization_name": { "value": "PGESKSE", "unit": "" }, + "configuration_summary": { "value": "PGESKSE", "unit": "" }, + "best_figure_of_merit": { "value": "0.25293", "unit": "" }, + "brilliance_transfer": { "value": "0.47344", "unit": "" }, + "event_file_name_suffix": { "value": "4Hsize_3moderator_size_y", "unit": "" }, + "number_of_parameters": { "value": 2, "unit": "" }, + "parameters_name": { "value": "Hsize , moderator_size_y", "unit": "" }, + "event_writen_present": { "value": true, "unit": "" }, + "event_writen_file": { "value": "master_record-writen_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_writen_timestamp": { "value": "2014-01-23T19:52:38", "unit": "" }, + "event_done_present": { "value": true, "unit": "" }, + "event_done_file": { "value": "master_record-done_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_done_timestamp": { "value": "2014-01-25T00:35:55", "unit": "" }, + "event_analysis_present": { "value": true, "unit": "" }, + "event_analysis_file": { "value": "output/analysis/master_record-analyzed_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_analysis_timestamp": { "value": "2014-01-28T14:03:02", "unit": "" }, + "dataset_name": { "value": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", "unit": "" }, + "run_name": { "value": "CAMEA CAMEA31", "unit": "" }, + "scan_name": { "value": "4Hsize_3moderator_size_y", "unit": "" }, + "output_file_name_base": { "value": "PGESKSE_4Hsize_3moderator_size_y", "unit": "" }, + "dataset_access_path": { "value": "/mnt/data/simulation/CAMEA/CAMEA31", "unit": "" }, + "parameters_structure": { "value": "[{\"name\": \"Hsize\", \"value\": \"1.5\", \"index\": \"4\"}, {\"name\": \"moderator_size_y\", \"value\": \"0.03\", \"index\": \"3\"}]", "unit": "" }, + "Hsize": { "value": 4, "unit": "cm" }, + "moderator_size_y": { "value": 3, "unit": "m" } + }, + "techniques": [ + { + "pid": "fe888574-5cc0-11ec-90c3-bf82943dec35", + "name": "Simulation" + } + ], + "size": 68386784, + "instrumentId": "" + }, + "orig_datablock": { + "size": 68386784, + "dataFileList": [ + { + "path": "launch_all.sh", + "size": 10171, + "time": "2014-01-23T19:52:37.000Z" + }, + { + "path": "suggested_reruns-fails.sh", + "size": 448, + "time": "2014-01-23T19:53:04.000Z" + }, + { + "path": "compile_all_py.sh", + "size": 273, + "time": "2014-01-23T19:52:37.000Z" + }, + { + "path": "clean3.sh", + "size": 354, + "time": "2014-01-25T10:44:54.000Z" + }, + { + "path": "master_record-done_4Hsize_3moderator_size_y.txt", + "size": 579, + "time": "2014-01-25T00:35:55.000Z" + }, + { + "path": "master_record-writen_4Hsize_3moderator_size_y.txt", + "size": 561, + "time": "2014-01-23T19:52:38.000Z" + }, + { + "path": "compile_all.sh", + "size": 259, + "time": "2014-01-23T19:52:37.000Z" + }, + { + "path": "output/brill_ref/brilliance_ref_4Hsize_3moderator_size_y.mat", + "size": 11624010, + "time": "2014-01-24T07:56:45.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_acceptance_ess.png", + "size": 521132, + "time": "2014-01-27T11:38:06.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_acceptance_pure.png", + "size": 518423, + "time": "2014-01-27T11:37:52.000Z" + }, + { + "path": "output/analysis/master_record-analyzed_4Hsize_3moderator_size_y.txt", + "size": 587, + "time": "2014-01-28T14:03:02.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_overall_pure.png", + "size": 144605, + "time": "2014-01-27T11:37:49.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_posdiv_ess.png", + "size": 336496, + "time": "2014-01-27T11:38:04.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y_all.mat", + "size": 34321077, + "time": "2014-01-25T00:35:55.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_overall_ess.png", + "size": 127660, + "time": "2014-01-27T11:38:02.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_geometry.dat", + "size": 2175, + "time": "2014-01-25T00:23:10.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y_ifit_analyse.m", + "size": 19482, + "time": "2014-01-23T19:52:40.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_geometry.png", + "size": 76259, + "time": "2014-01-27T11:38:09.000Z" + }, + { + "path": "output/analysis/PGESKSE_4Hsize_3moderator_size_y1_posdiv_pure.png", + "size": 353828, + "time": "2014-01-27T11:37:50.000Z" + }, + { + "path": "brilliance_refference/brilliance_ifit_4Hsize_3moderator_size_y.m", + "size": 3048, + "time": "2014-01-23T19:52:33.000Z" + }, + { + "path": "brilliance_refference/brilliance_4Hsize_3moderator_size_y1.mat", + "size": 11626979, + "time": "2014-01-24T07:56:42.000Z" + }, + { + "path": "brilliance_refference/brilliance_4Hsize_3moderator_size_y.batch", + "size": 671, + "time": "2014-01-23T19:52:32.000Z" + }, + { + "path": "brilliance_refference/input_used_4Hsize_3moderator_size_y.txt", + "size": 358, + "time": "2014-01-23T19:52:35.000Z" + }, + { + "path": "brilliance_refference/run_brilliance_ifit_4Hsize_3moderator_size_y.m", + "size": 53, + "time": "2014-01-23T19:52:36.000Z" + }, + { + "path": "PGESKSE/PGESKSE_4Hsize_3moderator_size_y.batch", + "size": 734, + "time": "2014-01-23T19:52:48.000Z" + }, + { + "path": "PGESKSE/PGESKSE_4Hsize_3moderator_size_y_ifit.m", + "size": 11101, + "time": "2014-01-23T19:52:48.000Z" + }, + { + "path": "PGESKSE/err_PGESKSE_4Hsize_3moderator_size_y.txt", + "size": 0, + "time": "2014-01-24T21:13:29.000Z" + }, + { + "path": "PGESKSE/run_PGESKSE_4Hsize_3moderator_size_y_ifit.m", + "size": 50, + "time": "2014-01-23T19:52:51.000Z" + }, + { + "path": "PGESKSE/out_PGESKSE_4Hsize_3moderator_size_y.txt", + "size": 8681220, + "time": "2014-01-25T00:35:58.000Z" + }, + { + "path": "PGESKSE/compile_PGESKSE_py.sh", + "size": 558, + "time": "2014-01-23T19:52:45.000Z" + }, + { + "path": "PGESKSE/compile_PGESKSE.sh", + "size": 540, + "time": "2014-01-23T19:52:45.000Z" + }, + { + "path": "PGESKSE/PGESKSE_4Hsize_3moderator_size_y1.par", + "size": 918, + "time": "2014-01-25T00:35:55.000Z" + }, + { + "path": "PGESKSE/PGESKSE_4Hsize_3moderator_size_y1_geometry.dat", + "size": 2175, + "time": "2014-01-25T00:23:10.000Z" + } + ] + }, + "ownable": { + "ownerGroup": "ess", + "accessGroups": ["dmsc"] + } +} + \ No newline at end of file diff --git a/examples/data/ingestion_simulation_dataset_ess_dataset.json b/examples/data/ingestion_simulation_dataset_ess_raw_dataset.json similarity index 99% rename from examples/data/ingestion_simulation_dataset_ess_dataset.json rename to examples/data/ingestion_simulation_dataset_ess_raw_dataset.json index ebd88bd..ae2977e 100644 --- a/examples/data/ingestion_simulation_dataset_ess_dataset.json +++ b/examples/data/ingestion_simulation_dataset_ess_raw_dataset.json @@ -1,7 +1,7 @@ { "id": "0275d813-be6b-444f-812f-b8311d129361", "dataset": { - "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", + "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE raw", "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", "creationLocation": "DMSC", "principalInvestigator": "Max Novelli", diff --git a/pyscicat/client.py b/pyscicat/client.py index 89397f1..7f96f2f 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -152,8 +152,12 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): # err = resp.json()["error"] # raise ScicatCommError(f"Error creating Sample {err}") + def upload_dataset(self, dataset: Dataset) -> str: - """Upload a raw or derived dataset (method is autosensing) + """ + Upload a raw or derived dataset (method is autosensing) + This function has been renamed as upsert. + WE are keeping this implementation for backward compatibility Parameters ---------- @@ -170,6 +174,25 @@ def upload_dataset(self, dataset: Dataset) -> str: ScicatCommError Raises if a non-20x message is returned """ + return self.upsert_dataset(dataset) + + + def upsert_dataset(self, dataset: Dataset) -> str: + """ + Create a new dataset or update an existing one + + + Parameters + ---------- + dataset : Dataset + Dataset to create or update + + Returns + ------- + str + pid of the dataset + """ + if isinstance(dataset, RawDataset): dataset_url = self._base_url + "RawDataSets/replaceOrCreate" elif isinstance(dataset, DerivedDataset): @@ -186,10 +209,39 @@ def upload_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + + def upload_new_dataset(self, dataset: Dataset) -> str: """ Upload a new dataset. Uses the generic dataset endpoint. - Relys on the endpoint to sense wthe dataset type + Relys on the endpoint to sense the dataset type + This function has been renamed. + We are keeping this implementation for backward compatibility + + Parameters + ---------- + dataset : Dataset + Dataset to create + + Returns + ------- + dataset : Dataset + Dataset created including the pid (or unique identifier) of the newly created dataset + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + return self.create_dataset(dataset) + + + def create_dataset(self, dataset: Dataset) -> str: + """ + Upload a new dataset. Uses the generic dataset endpoint. + Relys on the endpoint to sense the dataset type + This function has been renamed. + We are keeping this implementation for backward compatibility Parameters ---------- @@ -218,8 +270,34 @@ def upload_new_dataset(self, dataset: Dataset) -> str: return resp.json() + def upload_raw_dataset(self, dataset: Dataset) -> str: - """Upload a raw dataset + """ + Upload a raw dataset + This function has been renamed. + We are keeping this implementation for backward compatibility + + Parameters + ---------- + dataset : Dataset + Dataset to load + + Returns + ------- + str + pid (or unique identifier) of the newly created dataset + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + return self.upsert_raw_dataset(dataset) + + + def upsert_raw_dataset(self, dataset: Dataset) -> str: + """ + Create a new raw dataset or update an existing one Parameters ---------- @@ -245,8 +323,34 @@ def upload_raw_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + def upload_derived_dataset(self, dataset: Dataset) -> str: - """Upload a derived dataset + """ + Upload a derived dataset + This function has been renamed. + We are keeping this implementation for backward compatibility + + Parameters + ---------- + dataset : Dataset + Dataset to upload + + Returns + ------- + str + pid (or unique identifier) of the newly created dataset + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + return self.upsert_derived_dataset(dataset) + + + def upsert_derived_dataset(self, dataset: Dataset) -> str: + """ + Create a new derived dataset or update an existing one Parameters ---------- @@ -274,8 +378,12 @@ def upload_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): - """Upload a Datablock + """ + Upload a Datablock + This function has been renamed + We are keeping this implementation for backward compatibility Parameters ---------- @@ -292,7 +400,29 @@ def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets ScicatCommError Raises if a non-20x message is returned """ + return self.create_dataset_datablock(datablock,datasetType) + + + def create_dataset_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): + """ + create a new datablock for a dataset. + The dataset can be both Raw or Derived + + Parameters + ---------- + datablock : Datablock + Datablock to upload + + Returns + ------- + datablock : Datablock + The created Datablock with id + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ url = ( self._base_url + f"{datasetType}/{urllib.parse.quote_plus(datablock.datasetId)}/origdatablocks" @@ -304,10 +434,11 @@ def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets return resp.json() - def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: - """ - Post SciCat Dataset OrigDatablock + #def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: + """ + Create a new SciCat Dataset OrigDatablock + Parameters ---------- origdatablock : @@ -324,7 +455,32 @@ def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: Raises if a non-20x message is returned """ + return self.create_dataset_origdatabloack(origdatablock) + + + def create_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: + """ + Create a new SciCat Dataset OrigDatablock + This function has been renamed. + It is still accessible with the original name for backward compatibility + The original name is upload_dataset_origdatablock + + Parameters + ---------- + origdatablock : + The OrigDatablock to create + Returns + ------- + dict + The created OrigDatablock with id + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + + """ encoded_pid = urllib.parse.quote_plus(origdatablock.datasetId) endpoint = "Datasets/" + encoded_pid + "/origdatablocks" url = self._base_url + endpoint @@ -336,10 +492,47 @@ def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: return resp.json() + """ + Create a new SciCat Dataset OrigDatablock + Original name, kept for for backward compatibility + """ + upload_dataset_origdatablock = create_dataset_origdatablock + + def upload_attachment( - self, attachment: Attachment, datasetType: str = "RawDatasets" + self, + attachment: Attachment, + datasetType: str = "RawDatasets" ): - """Upload an Attachment. Note that datasetType can be provided to determine the type of dataset + """ + Upload an Attachment. + Note that datasetType can be provided to determine the type of dataset + that this attachment is attached to. This is required for creating the url that SciCat uses. + THis function has been renamed. + WE are kleeping this implementation for backward compatibility + + Parameters + ---------- + attachment : Attachment + Attachment to upload + + datasetType : str + Type of dataset to upload to, default is `RawDatasets` + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + return self.create_dataset_attachment(attachment,datasetType) + + def create_dataset_attachment( + self, + attachment: Attachment, + datasetType: str = "RawDatasets" + ): + """ + Create a new Attachment for a dataset. + Note that datasetType can be provided to determine the type of dataset that this attachment is attached to. This is required for creating the url that SciCat uses. Parameters @@ -371,6 +564,7 @@ def upload_attachment( err = resp.json()["error"] raise ScicatCommError(f"Error uploading thumbnail. {err}") + def get_datasets_full_query(self, skip=0, limit=25, query_fields=None): """Gets datasets using the fullQuery mechanism of SciCat. This is appropriate for cases where might want paging and cases where you want to perform @@ -447,7 +641,7 @@ def get_published_data(self, filter=None) -> List[PublishedData]: ```python filter = {"doi": "1234"} ``` - + Parameters ---------- filter : dict @@ -458,10 +652,8 @@ def get_published_data(self, filter=None) -> List[PublishedData]: else: filter = json.dumps(filter) - url = f'{self._base_url}/PublishedData' + ( - f'?filter={{"where":{filter}}}' - if filter - else '' + url = f"{self._base_url}/PublishedData" + ( + f'?filter={{"where":{filter}}}' if filter else "" ) response = self._send_to_scicat(url, cmd="get") if not response.ok: @@ -470,7 +662,6 @@ def get_published_data(self, filter=None) -> List[PublishedData]: return None return response.json() - def get_dataset_by_pid(self, pid=None) -> Dataset: """Gets dataset with the pid provided. diff --git a/pyscicat/model.py b/pyscicat/model.py index d354f36..bc9451a 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -221,7 +221,7 @@ class Attachment(Ownable): datasetId: str -class PublishedData(): +class PublishedData: """ Published Data with registered DOI """ diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 39ba2ca..8ce83fc 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -40,7 +40,7 @@ def add_mock_requests(mock_request): mock_request.post(local_url + "Datasets", json={"pid": "17"}) -def test_scicate_ingest(): +def test_scicat_ingest(): with requests_mock.Mocker() as mock_request: add_mock_requests(mock_request) scicat = from_credentials( diff --git a/pyscicat/tests/test_new_dataset.py b/pyscicat/tests/test_new_dataset.py index e530db4..dafbc6a 100644 --- a/pyscicat/tests/test_new_dataset.py +++ b/pyscicat/tests/test_new_dataset.py @@ -12,46 +12,117 @@ Ownable, ) -global test_dataset +global test_datasets local_url = "http://localhost:3000/api/v3/" -test_dataset_file = "../../examples/data/ingestion_simulation_dataset_ess_dataset.json" -test_dataset = None +test_dataset_files = { + 'raw' : "../../examples/data/ingestion_simulation_dataset_ess_raw_dataset.json", + 'derived' : "../../examples/data/ingestion_simulation_dataset_ess_derived_dataset.json" +} +test_datasets = {} def set_up_test_environment(mock_request): - global test_dataset + global test_datasets # load test data - data_file_path = Path(__file__).parent.joinpath(test_dataset_file).resolve() - with open(data_file_path, "r") as fh: - test_dataset = json.load(fh) + for name, path in test_dataset_files.items(): + data_file_path = Path(__file__).parent.joinpath(path).resolve() + with open(data_file_path, "r") as fh: + test_datasets[name] = json.load(fh) mock_request.post( local_url + "Users/login", json={"id": "a_token"}, ) +def set_up_mock_raw_dataset(mock_request): + data = test_datasets['raw'] + mock_request.post( local_url + "Datasets", - json={**{"pid": test_dataset["id"]}, **test_dataset["dataset"]}, + json={**{"pid": data["id"]}, **data["dataset"]}, ) - encoded_pid = urllib.parse.quote_plus(test_dataset["id"]) + encoded_pid = urllib.parse.quote_plus(data["id"]) mock_request.post( local_url + "Datasets/" + encoded_pid + "/origdatablocks", json={ - "size": test_dataset["orig_datablock"]["size"], - "datasetId": test_dataset["id"], - "dataFileList": test_dataset["orig_datablock"]["dataFileList"], + "size": data["orig_datablock"]["size"], + "datasetId": data["id"], + "dataFileList": data["orig_datablock"]["dataFileList"], }, ) + return data + + +def set_up_mock_derived_dataset(mock_request): + data = test_datasets['derived'] + + mock_request.post( + local_url + "Datasets", + json={**{"pid": data["id"]}, **data["dataset"]}, + ) + + encoded_pid = urllib.parse.quote_plus(data["id"]) + mock_request.post( + local_url + "Datasets/" + encoded_pid + "/origdatablocks", + json={ + "size": data["orig_datablock"]["size"], + "datasetId": data["id"], + "dataFileList": data["orig_datablock"]["dataFileList"], + }, + ) + + return data + + +def test_scicat_ingest_raw_dataset(): + with requests_mock.Mocker() as mock_request: + set_up_test_environment(mock_request) + data = set_up_mock_raw_dataset(mock_request) + scicat = ScicatClient( + base_url=local_url, + username="Zaphod", + password="heartofgold", + ) + assert ( + scicat._token == "a_token" + ), "scicat client set the token given by the server" + + ownable = Ownable(**data['ownable']) + + # Create Dataset + dataset = RawDataset( + **data["dataset"], + **ownable.dict() + ) + created_dataset = scicat.create_dataset(dataset) + + assert created_dataset["pid"] == data["id"] -def test_scicate_ingest_raw_dataset(): + # origDatablock with DataFiles + origDataBlock = OrigDatablock( + size=data["orig_datablock"]["size"], + datasetId=created_dataset["pid"], + dataFileList=[ + DataFile(**file) + for file in data["orig_datablock"]["dataFileList"] + ], + **ownable.dict() + ) + created_origdatablock = scicat.create_dataset_origdatablock(origDataBlock) + assert len(created_origdatablock["dataFileList"]) == len( + data["orig_datablock"]["dataFileList"] + ) + + +def test_scicat_ingest_derived_dataset(): with requests_mock.Mocker() as mock_request: set_up_test_environment(mock_request) + data = set_up_mock_derived_dataset(mock_request) scicat = ScicatClient( base_url=local_url, username="Zaphod", @@ -61,25 +132,28 @@ def test_scicate_ingest_raw_dataset(): scicat._token == "a_token" ), "scicat client set the token given by the server" - ownable = Ownable(ownerGroup="magrathea", accessGroups=["deep_though"]) + ownable = Ownable(**data['ownable']) # Create Dataset - dataset = RawDataset(**test_dataset["dataset"], **ownable.dict()) - created_dataset = scicat.upload_new_dataset(dataset) + dataset = RawDataset( + **data["dataset"], + **ownable.dict() + ) + created_dataset = scicat.create_dataset(dataset) - assert created_dataset["pid"] == test_dataset["id"] + assert created_dataset["pid"] == data["id"] # origDatablock with DataFiles origDataBlock = OrigDatablock( - size=test_dataset["orig_datablock"]["size"], + size=data["orig_datablock"]["size"], datasetId=created_dataset["pid"], dataFileList=[ DataFile(**file) - for file in test_dataset["orig_datablock"]["dataFileList"] + for file in data["orig_datablock"]["dataFileList"] ], **ownable.dict() ) - created_origdatablock = scicat.upload_dataset_origdatablock(origDataBlock) + created_origdatablock = scicat.create_dataset_origdatablock(origDataBlock) assert len(created_origdatablock["dataFileList"]) == len( - test_dataset["orig_datablock"]["dataFileList"] + data["orig_datablock"]["dataFileList"] ) From a9edeab88d09303f8fa8b9d3d9263715929d2693 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Fri, 3 Jun 2022 13:47:35 +0200 Subject: [PATCH 34/98] Fixed naming convention on few functions, added test on published data --- examples/data/published_data.json | 56 ++++ pyscicat/client.py | 270 ++++++------------ .../{test_new_dataset.py => test_suite_2.py} | 37 ++- 3 files changed, 185 insertions(+), 178 deletions(-) create mode 100644 examples/data/published_data.json rename pyscicat/tests/{test_new_dataset.py => test_suite_2.py} (82%) diff --git a/examples/data/published_data.json b/examples/data/published_data.json new file mode 100644 index 0000000..ab2680d --- /dev/null +++ b/examples/data/published_data.json @@ -0,0 +1,56 @@ +[ + { + "doi": "10.17199/03dd9804-1b04-4d36-b0fb-cf66e9891e7d", + "affiliation": "ESS", + "creator": [ + "Oliver Lohmann" + ], + "publisher": "ESS", + "publicationYear": 2019, + "title": "SANS/Reflectometry", + "url": "", + "abstract": "SANS/Reflectometry", + "dataDescription": "https://github.com/ess-dmsc/ess_file_formats/wiki/NeXus", + "resourceType": "NeXus HDF5", + "numberOfFiles": null, + "sizeOfArchive": null, + "pidArray": [ + "20.500.12269/0a269002-83e2-4f18-bb98-36c01836d66a" + ], + "authors": [ + "Oliver Lohmann" + ], + "registeredTime": "2020-09-01T14:16:15.552Z", + "status": "registered", + "thumbnail": "", + "createdBy": "admin", + "updatedBy": "admin", + "createdAt": "2020-01-03T19:38:34.203Z", + "updatedAt": "2020-09-09T09:37:58.023Z" + }, + { + "doi": "10.17199/165f8a52-c15d-4c96-ad7d-fb0cbe969f66", + "creator": [ + "Peter Kadletz" + ], + "publisher": "ESS", + "publicationYear": 2020, + "title": "Final bte", + "url": "", + "abstract": "Peter Kadletz, Tobias Richter", + "dataDescription": "https://github.com/ess-dmsc/ess_file_formats/wiki/NeXus", + "resourceType": "raw", + "numberOfFiles": null, + "sizeOfArchive": null, + "pidArray": [ + "20.500.12269/2511nicos_00002511.hdf" + ], + "registeredTime": "2020-09-01T14:16:17.272Z", + "status": "registered", + "scicatUser": "ingestor", + "thumbnail": "", + "updatedBy": "admin", + "createdAt": "2022-06-03T11:16:09.681Z", + "updatedAt": "2020-09-09T09:37:58.094Z" + } +] \ No newline at end of file diff --git a/pyscicat/client.py b/pyscicat/client.py index 7f96f2f..492cde5 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -153,34 +153,13 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): # raise ScicatCommError(f"Error creating Sample {err}") - def upload_dataset(self, dataset: Dataset) -> str: - """ - Upload a raw or derived dataset (method is autosensing) - This function has been renamed as upsert. - WE are keeping this implementation for backward compatibility - - Parameters - ---------- - dataset : Dataset - Dataset to load - - Returns - ------- - str - pid (or unique identifier) of the newly created dataset - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - return self.upsert_dataset(dataset) - - def upsert_dataset(self, dataset: Dataset) -> str: """ Create a new dataset or update an existing one - + This function was renamed. + It is still accessible with the original name for backward compatibility + The original name was upload_dataset + Parameters ---------- @@ -209,39 +188,20 @@ def upsert_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + """ + Upload or create a new dataset + Original name, kept for for backward compatibility + """ + upload_dataset = upsert_dataset - def upload_new_dataset(self, dataset: Dataset) -> str: - """ - Upload a new dataset. Uses the generic dataset endpoint. - Relys on the endpoint to sense the dataset type - This function has been renamed. - We are keeping this implementation for backward compatibility - - Parameters - ---------- - dataset : Dataset - Dataset to create - - Returns - ------- - dataset : Dataset - Dataset created including the pid (or unique identifier) of the newly created dataset - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - return self.create_dataset(dataset) - - def create_dataset(self, dataset: Dataset) -> str: """ Upload a new dataset. Uses the generic dataset endpoint. - Relys on the endpoint to sense the dataset type - This function has been renamed. - We are keeping this implementation for backward compatibility + Relies on the endpoint to sense the dataset type + This function was renamed. + It is still accessible with the original name for backward compatibility + The original name was upload_new_dataset Parameters ---------- @@ -270,34 +230,21 @@ def create_dataset(self, dataset: Dataset) -> str: return resp.json() - - def upload_raw_dataset(self, dataset: Dataset) -> str: - """ - Upload a raw dataset - This function has been renamed. - We are keeping this implementation for backward compatibility - - Parameters - ---------- - dataset : Dataset - Dataset to load - - Returns - ------- - str - pid (or unique identifier) of the newly created dataset - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - return self.upsert_raw_dataset(dataset) + """ + Upload a new dataset + Original name, kept for for backward compatibility + """ + upload_new_dataset = create_dataset + + def upsert_raw_dataset(self, dataset: Dataset) -> str: """ Create a new raw dataset or update an existing one + This function was renamed. + It is still accessible with the original name for backward compatibility + The original name was upload_raw_dataset Parameters ---------- @@ -324,33 +271,20 @@ def upsert_raw_dataset(self, dataset: Dataset) -> str: return new_pid - def upload_derived_dataset(self, dataset: Dataset) -> str: - """ - Upload a derived dataset - This function has been renamed. - We are keeping this implementation for backward compatibility - - Parameters - ---------- - dataset : Dataset - Dataset to upload - - Returns - ------- - str - pid (or unique identifier) of the newly created dataset + """ + Upload a raw dataset + Original name, kept for for backward compatibility + """ + upload_raw_dataset = upsert_raw_dataset - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - return self.upsert_derived_dataset(dataset) def upsert_derived_dataset(self, dataset: Dataset) -> str: """ Create a new derived dataset or update an existing one + This function was renamed. + It is still accessible with the original name for backward compatibility + The original name was upsert_derived_dataset Parameters ---------- @@ -378,35 +312,19 @@ def upsert_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid - - def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): - """ - Upload a Datablock - This function has been renamed - We are keeping this implementation for backward compatibility - - Parameters - ---------- - datablock : Datablock - Datablock to upload - - Returns - ------- - datablock : Datablock - The created Datablock with id - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - return self.create_dataset_datablock(datablock,datasetType) + """ + Upload a derived dataset + Original name, kept for for backward compatibility + """ + upload_derived_dataset = upsert_derived_dataset def create_dataset_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): """ - create a new datablock for a dataset. - The dataset can be both Raw or Derived + Create a new datablock for a dataset. + The dataset can be both Raw or Derived. + It is still accessible with the original name for backward compatibility + The original name was upload_datablock Parameters ---------- @@ -434,36 +352,20 @@ def create_dataset_datablock(self, datablock: Datablock, datasetType: str = "Raw return resp.json() - - #def upload_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: - """ - Create a new SciCat Dataset OrigDatablock - - Parameters - ---------- - origdatablock : - The OrigDatablock to create - - Returns - ------- - dict - The created OrigDatablock with id - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - - """ - return self.create_dataset_origdatabloack(origdatablock) + """ + Upload a Datablock + Original name, kept for for backward compatibility + """ + upload_datablock = create_dataset_datablock + def create_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: """ Create a new SciCat Dataset OrigDatablock This function has been renamed. It is still accessible with the original name for backward compatibility - The original name is upload_dataset_origdatablock + The original name was upload_dataset_origdatablock Parameters ---------- @@ -499,31 +401,6 @@ def create_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: upload_dataset_origdatablock = create_dataset_origdatablock - def upload_attachment( - self, - attachment: Attachment, - datasetType: str = "RawDatasets" - ): - """ - Upload an Attachment. - Note that datasetType can be provided to determine the type of dataset - that this attachment is attached to. This is required for creating the url that SciCat uses. - THis function has been renamed. - WE are kleeping this implementation for backward compatibility - - Parameters - ---------- - attachment : Attachment - Attachment to upload - - datasetType : str - Type of dataset to upload to, default is `RawDatasets` - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - return self.create_dataset_attachment(attachment,datasetType) def create_dataset_attachment( self, @@ -534,6 +411,9 @@ def create_dataset_attachment( Create a new Attachment for a dataset. Note that datasetType can be provided to determine the type of dataset that this attachment is attached to. This is required for creating the url that SciCat uses. + This function has been renamed. + It is still accessible with the original name for backward compatibility + The original name was upload_attachment Parameters ---------- @@ -564,9 +444,16 @@ def create_dataset_attachment( err = resp.json()["error"] raise ScicatCommError(f"Error uploading thumbnail. {err}") + """ + Create a new attachement for a dataset + Original name, kept for for backward compatibility + """ + upload_attachment = create_dataset_attachment + - def get_datasets_full_query(self, skip=0, limit=25, query_fields=None): - """Gets datasets using the fullQuery mechanism of SciCat. This is + def find_datasets_full_query(self, skip=0, limit=25, query_fields=None): + """ + Gets datasets using the fullQuery mechanism of SciCat. This is appropriate for cases where might want paging and cases where you want to perform a text search on the Datasets collection. The full features of fullQuery search are beyond this document. @@ -576,6 +463,10 @@ def get_datasets_full_query(self, skip=0, limit=25, query_fields=None): To query based on the full text search, send `{"text": " List[Dataset]: - """Gets datasets using the simple fiter mechanism. This + """ + find a set of datasets according the full query provided + Original name, kept for for backward compatibility + """ + get_datasets_full_query = find_datasets_full_query + + + + def find_datasets(self, filter_fields=None) -> List[Dataset]: + """ + Gets datasets using the simple fiter mechanism. This is appropriate when you do not require paging or text search, but want to be able to limit results based on items in the Dataset object. @@ -632,8 +532,16 @@ def get_datasets(self, filter_fields=None) -> List[Dataset]: return None return response.json() - def get_published_data(self, filter=None) -> List[PublishedData]: - """Gets published data using the simple fiter mechanism. This + """ + find a set of datasets according to the simple filter provided + Original name, kept for for backward compatibility + """ + get_datasets = find_datasets + + + def find_published_data(self, filter=None) -> List[PublishedData]: + """ + retrieve all the published data using the simple fiter mechanism. This is appropriate when you do not require paging or text search, but want to be able to limit results based on items in the Dataset object. @@ -652,9 +560,10 @@ def get_published_data(self, filter=None) -> List[PublishedData]: else: filter = json.dumps(filter) - url = f"{self._base_url}/PublishedData" + ( + url = f"{self._base_url}PublishedData" + ( f'?filter={{"where":{filter}}}' if filter else "" ) + print(url) response = self._send_to_scicat(url, cmd="get") if not response.ok: err = response.json()["error"] @@ -662,6 +571,13 @@ def get_published_data(self, filter=None) -> List[PublishedData]: return None return response.json() + """ + find a set of published data according to the simple filter provided + Original name, kept for for backward compatibility + """ + get_published_data = find_published_data + + def get_dataset_by_pid(self, pid=None) -> Dataset: """Gets dataset with the pid provided. diff --git a/pyscicat/tests/test_new_dataset.py b/pyscicat/tests/test_suite_2.py similarity index 82% rename from pyscicat/tests/test_new_dataset.py rename to pyscicat/tests/test_suite_2.py index dafbc6a..1a36106 100644 --- a/pyscicat/tests/test_new_dataset.py +++ b/pyscicat/tests/test_suite_2.py @@ -17,7 +17,8 @@ local_url = "http://localhost:3000/api/v3/" test_dataset_files = { 'raw' : "../../examples/data/ingestion_simulation_dataset_ess_raw_dataset.json", - 'derived' : "../../examples/data/ingestion_simulation_dataset_ess_derived_dataset.json" + 'derived' : "../../examples/data/ingestion_simulation_dataset_ess_derived_dataset.json", + 'published_data' : "../../examples/data/published_data.json" } test_datasets = {} @@ -37,6 +38,7 @@ def set_up_test_environment(mock_request): json={"id": "a_token"}, ) + def set_up_mock_raw_dataset(mock_request): data = test_datasets['raw'] @@ -79,6 +81,19 @@ def set_up_mock_derived_dataset(mock_request): return data +def set_up_mock_published_data(mock_request): + data = test_datasets['published_data'] + + mock_url = local_url + "PublishedData" + print("Mock : " + mock_url) + mock_request.get( + mock_url, + json=data, + ) + + return data + + def test_scicat_ingest_raw_dataset(): with requests_mock.Mocker() as mock_request: set_up_test_environment(mock_request) @@ -157,3 +172,23 @@ def test_scicat_ingest_derived_dataset(): assert len(created_origdatablock["dataFileList"]) == len( data["orig_datablock"]["dataFileList"] ) + + +def test_scicat_find_published_data(): + with requests_mock.Mocker() as mock_request: + set_up_test_environment(mock_request) + data = set_up_mock_published_data(mock_request) + scicat = ScicatClient( + base_url=local_url, + username="Zaphod", + password="heartofgold", + ) + assert ( + scicat._token == "a_token" + ), "scicat client set the token given by the server" + + returned_data = scicat.find_published_data() + + assert len(data) == len(returned_data) + assert data == returned_data + From 479d41dc069bf95e20cca0dead136173e9b68e09 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Fri, 3 Jun 2022 14:34:37 +0200 Subject: [PATCH 35/98] fixed linting and code styling --- ...imulation_dataset_ess_derived_dataset.json | 131 +++++++++-------- ...on_simulation_dataset_ess_raw_dataset.json | 137 +++++++++--------- examples/data/published_data.json | 2 +- pyscicat/client.py | 30 +--- pyscicat/tests/test_suite_2.py | 33 ++--- requirements-hdf5.txt | 2 +- requirements.txt | 2 +- 7 files changed, 155 insertions(+), 182 deletions(-) diff --git a/examples/data/ingestion_simulation_dataset_ess_derived_dataset.json b/examples/data/ingestion_simulation_dataset_ess_derived_dataset.json index 1b0bcea..53c18b4 100644 --- a/examples/data/ingestion_simulation_dataset_ess_derived_dataset.json +++ b/examples/data/ingestion_simulation_dataset_ess_derived_dataset.json @@ -1,9 +1,9 @@ { - "id": "9be3bd96-e256-11ec-bd08-f32122965a87", + "id": "9be3bd96-e256-11ec-bd08-f32122965a87", "dataset": { - "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE derived", - "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", - "investigator": "Max Novelli", + "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE derived", + "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", + "investigator": "Max Novelli", "inputDatasets" : ["0275d813-be6b-444f-812f-b8311d129361"], "usedSoftware" : ["python","My software"], "jobParameters" : { @@ -11,76 +11,76 @@ "parameter-2" : "value-2" }, "jobLogData" : "Some jebrish about the dataset", - "owner": "Massimiliano Novelli", - "ownerEmail": "max.novelli@ess.eu", - "contactEmail": "max.novelli@ess.eu", - "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", - "creationTime": "2022-03-07T15:44:59.000Z", - "type": "derived", + "owner": "Massimiliano Novelli", + "ownerEmail": "max.novelli@ess.eu", + "contactEmail": "max.novelli@ess.eu", + "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", + "creationTime": "2022-03-07T15:44:59.000Z", + "type": "derived", "scientificMetadata": { - "sample_width": { "value": 0.015, "unit": "m" }, - "sample_height": { "value": 0.015, "unit": "m" }, - "divergence_requirement_horizontal": { "value": 0.75, "unit": "deg" }, - "divergence_requirement_vertical": { "value": 1, "unit": "deg" }, - "guide_sample_distance": { "value": 0.6, "unit": "m" }, - "lower_wavelength_limit": { "value": 1, "unit": "\u00c5" }, - "upper_wavelength_limit": { "value": 3.6, "unit": "\u00c5" }, - "moderator_width": { "value": 0.12, "unit": "m" }, - "moderator_height": { "value": 0.03, "unit": "m" }, - "moderator_sample_distance": { "value": 170, "unit": "m" }, - "parsing_variables": { "value": "guide_start , startx1 , starty1 , length1", "unit": "" }, - "parsing_min_guide_start": { "value": 2.000035881054106, "unit": "m" }, - "parsing_max_guide_start": { "value": 5.407538318585075, "unit": "m" }, - "parsing_mean_guide_start": { "value": 2.3475508029429557, "unit": "m" }, - "parsing_std_guide_start": { "value": 0.5522363822422368, "unit": "m" }, - "parsing_min_startx1": { "value": 0.006706596967962139, "unit": "m" }, - "parsing_max_startx1": { "value": 0.1460959338571846, "unit": "m" }, - "parsing_mean_startx1": { "value": 0.08885675463366878, "unit": "m" }, - "parsing_std_startx1": { "value": 0.017699812942929365, "unit": "m" }, - "parsing_min_starty1": { "value": 0.011762187831963904, "unit": "m" }, - "parsing_max_starty1": { "value": 0.14999127413576652, "unit": "m" }, - "parsing_mean_starty1": { "value": 0.13009670276273638, "unit": "m" }, - "parsing_std_starty1": { "value": 0.011522927034872269, "unit": "m" }, - "parsing_min_length1": { "value": 28.915197821153896, "unit": "" }, - "parsing_max_length1": { "value": 95.07944574028325, "unit": "" }, - "parsing_mean_length1": { "value": 64.23126877070395, "unit": "" }, - "parsing_std_length1": { "value": 10.210341803833671, "unit": "" }, - "optimization_name": { "value": "PGESKSE", "unit": "" }, - "configuration_summary": { "value": "PGESKSE", "unit": "" }, - "best_figure_of_merit": { "value": "0.25293", "unit": "" }, - "brilliance_transfer": { "value": "0.47344", "unit": "" }, - "event_file_name_suffix": { "value": "4Hsize_3moderator_size_y", "unit": "" }, - "number_of_parameters": { "value": 2, "unit": "" }, - "parameters_name": { "value": "Hsize , moderator_size_y", "unit": "" }, - "event_writen_present": { "value": true, "unit": "" }, - "event_writen_file": { "value": "master_record-writen_4Hsize_3moderator_size_y.txt", "unit": "" }, - "event_writen_timestamp": { "value": "2014-01-23T19:52:38", "unit": "" }, - "event_done_present": { "value": true, "unit": "" }, - "event_done_file": { "value": "master_record-done_4Hsize_3moderator_size_y.txt", "unit": "" }, - "event_done_timestamp": { "value": "2014-01-25T00:35:55", "unit": "" }, - "event_analysis_present": { "value": true, "unit": "" }, + "sample_width": { "value": 0.015, "unit": "m" }, + "sample_height": { "value": 0.015, "unit": "m" }, + "divergence_requirement_horizontal": { "value": 0.75, "unit": "deg" }, + "divergence_requirement_vertical": { "value": 1, "unit": "deg" }, + "guide_sample_distance": { "value": 0.6, "unit": "m" }, + "lower_wavelength_limit": { "value": 1, "unit": "\u00c5" }, + "upper_wavelength_limit": { "value": 3.6, "unit": "\u00c5" }, + "moderator_width": { "value": 0.12, "unit": "m" }, + "moderator_height": { "value": 0.03, "unit": "m" }, + "moderator_sample_distance": { "value": 170, "unit": "m" }, + "parsing_variables": { "value": "guide_start , startx1 , starty1 , length1", "unit": "" }, + "parsing_min_guide_start": { "value": 2.000035881054106, "unit": "m" }, + "parsing_max_guide_start": { "value": 5.407538318585075, "unit": "m" }, + "parsing_mean_guide_start": { "value": 2.3475508029429557, "unit": "m" }, + "parsing_std_guide_start": { "value": 0.5522363822422368, "unit": "m" }, + "parsing_min_startx1": { "value": 0.006706596967962139, "unit": "m" }, + "parsing_max_startx1": { "value": 0.1460959338571846, "unit": "m" }, + "parsing_mean_startx1": { "value": 0.08885675463366878, "unit": "m" }, + "parsing_std_startx1": { "value": 0.017699812942929365, "unit": "m" }, + "parsing_min_starty1": { "value": 0.011762187831963904, "unit": "m" }, + "parsing_max_starty1": { "value": 0.14999127413576652, "unit": "m" }, + "parsing_mean_starty1": { "value": 0.13009670276273638, "unit": "m" }, + "parsing_std_starty1": { "value": 0.011522927034872269, "unit": "m" }, + "parsing_min_length1": { "value": 28.915197821153896, "unit": "" }, + "parsing_max_length1": { "value": 95.07944574028325, "unit": "" }, + "parsing_mean_length1": { "value": 64.23126877070395, "unit": "" }, + "parsing_std_length1": { "value": 10.210341803833671, "unit": "" }, + "optimization_name": { "value": "PGESKSE", "unit": "" }, + "configuration_summary": { "value": "PGESKSE", "unit": "" }, + "best_figure_of_merit": { "value": "0.25293", "unit": "" }, + "brilliance_transfer": { "value": "0.47344", "unit": "" }, + "event_file_name_suffix": { "value": "4Hsize_3moderator_size_y", "unit": "" }, + "number_of_parameters": { "value": 2, "unit": "" }, + "parameters_name": { "value": "Hsize , moderator_size_y", "unit": "" }, + "event_writen_present": { "value": true, "unit": "" }, + "event_writen_file": { "value": "master_record-writen_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_writen_timestamp": { "value": "2014-01-23T19:52:38", "unit": "" }, + "event_done_present": { "value": true, "unit": "" }, + "event_done_file": { "value": "master_record-done_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_done_timestamp": { "value": "2014-01-25T00:35:55", "unit": "" }, + "event_analysis_present": { "value": true, "unit": "" }, "event_analysis_file": { "value": "output/analysis/master_record-analyzed_4Hsize_3moderator_size_y.txt", "unit": "" }, - "event_analysis_timestamp": { "value": "2014-01-28T14:03:02", "unit": "" }, - "dataset_name": { "value": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", "unit": "" }, - "run_name": { "value": "CAMEA CAMEA31", "unit": "" }, - "scan_name": { "value": "4Hsize_3moderator_size_y", "unit": "" }, - "output_file_name_base": { "value": "PGESKSE_4Hsize_3moderator_size_y", "unit": "" }, - "dataset_access_path": { "value": "/mnt/data/simulation/CAMEA/CAMEA31", "unit": "" }, - "parameters_structure": { "value": "[{\"name\": \"Hsize\", \"value\": \"1.5\", \"index\": \"4\"}, {\"name\": \"moderator_size_y\", \"value\": \"0.03\", \"index\": \"3\"}]", "unit": "" }, - "Hsize": { "value": 4, "unit": "cm" }, + "event_analysis_timestamp": { "value": "2014-01-28T14:03:02", "unit": "" }, + "dataset_name": { "value": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", "unit": "" }, + "run_name": { "value": "CAMEA CAMEA31", "unit": "" }, + "scan_name": { "value": "4Hsize_3moderator_size_y", "unit": "" }, + "output_file_name_base": { "value": "PGESKSE_4Hsize_3moderator_size_y", "unit": "" }, + "dataset_access_path": { "value": "/mnt/data/simulation/CAMEA/CAMEA31", "unit": "" }, + "parameters_structure": { "value": "[{\"name\": \"Hsize\", \"value\": \"1.5\", \"index\": \"4\"}, {\"name\": \"moderator_size_y\", \"value\": \"0.03\", \"index\": \"3\"}]", "unit": "" }, + "Hsize": { "value": 4, "unit": "cm" }, "moderator_size_y": { "value": 3, "unit": "m" } - }, + }, "techniques": [ { - "pid": "fe888574-5cc0-11ec-90c3-bf82943dec35", + "pid": "fe888574-5cc0-11ec-90c3-bf82943dec35", "name": "Simulation" } - ], - "size": 68386784, + ], + "size": 68386784, "instrumentId": "" - }, + }, "orig_datablock": { - "size": 68386784, + "size": 68386784, "dataFileList": [ { "path": "launch_all.sh", @@ -254,4 +254,3 @@ "accessGroups": ["dmsc"] } } - \ No newline at end of file diff --git a/examples/data/ingestion_simulation_dataset_ess_raw_dataset.json b/examples/data/ingestion_simulation_dataset_ess_raw_dataset.json index ae2977e..c2b5817 100644 --- a/examples/data/ingestion_simulation_dataset_ess_raw_dataset.json +++ b/examples/data/ingestion_simulation_dataset_ess_raw_dataset.json @@ -1,82 +1,82 @@ { - "id": "0275d813-be6b-444f-812f-b8311d129361", + "id": "0275d813-be6b-444f-812f-b8311d129361", "dataset": { - "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE raw", - "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", - "creationLocation": "DMSC", - "principalInvestigator": "Max Novelli", - "owner": "Massimiliano Novelli", - "ownerEmail": "max.novelli@ess.eu", - "contactEmail": "max.novelli@ess.eu", - "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", - "creationTime": "2022-03-07T15:44:59.000Z", - "type": "raw", + "datasetName": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE raw", + "description": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", + "creationLocation": "DMSC", + "principalInvestigator": "Max Novelli", + "owner": "Massimiliano Novelli", + "ownerEmail": "max.novelli@ess.eu", + "contactEmail": "max.novelli@ess.eu", + "sourceFolder": "/mnt/data/simulation/CAMEA/CAMEA31", + "creationTime": "2022-03-07T15:44:59.000Z", + "type": "raw", "scientificMetadata": { - "sample_width": { "value": 0.015, "unit": "m" }, - "sample_height": { "value": 0.015, "unit": "m" }, - "divergence_requirement_horizontal": { "value": 0.75, "unit": "deg" }, - "divergence_requirement_vertical": { "value": 1, "unit": "deg" }, - "guide_sample_distance": { "value": 0.6, "unit": "m" }, - "lower_wavelength_limit": { "value": 1, "unit": "\u00c5" }, - "upper_wavelength_limit": { "value": 3.6, "unit": "\u00c5" }, - "moderator_width": { "value": 0.12, "unit": "m" }, - "moderator_height": { "value": 0.03, "unit": "m" }, - "moderator_sample_distance": { "value": 170, "unit": "m" }, - "parsing_variables": { "value": "guide_start , startx1 , starty1 , length1", "unit": "" }, - "parsing_min_guide_start": { "value": 2.000035881054106, "unit": "m" }, - "parsing_max_guide_start": { "value": 5.407538318585075, "unit": "m" }, - "parsing_mean_guide_start": { "value": 2.3475508029429557, "unit": "m" }, - "parsing_std_guide_start": { "value": 0.5522363822422368, "unit": "m" }, - "parsing_min_startx1": { "value": 0.006706596967962139, "unit": "m" }, - "parsing_max_startx1": { "value": 0.1460959338571846, "unit": "m" }, - "parsing_mean_startx1": { "value": 0.08885675463366878, "unit": "m" }, - "parsing_std_startx1": { "value": 0.017699812942929365, "unit": "m" }, - "parsing_min_starty1": { "value": 0.011762187831963904, "unit": "m" }, - "parsing_max_starty1": { "value": 0.14999127413576652, "unit": "m" }, - "parsing_mean_starty1": { "value": 0.13009670276273638, "unit": "m" }, - "parsing_std_starty1": { "value": 0.011522927034872269, "unit": "m" }, - "parsing_min_length1": { "value": 28.915197821153896, "unit": "" }, - "parsing_max_length1": { "value": 95.07944574028325, "unit": "" }, - "parsing_mean_length1": { "value": 64.23126877070395, "unit": "" }, - "parsing_std_length1": { "value": 10.210341803833671, "unit": "" }, - "optimization_name": { "value": "PGESKSE", "unit": "" }, - "configuration_summary": { "value": "PGESKSE", "unit": "" }, - "best_figure_of_merit": { "value": "0.25293", "unit": "" }, - "brilliance_transfer": { "value": "0.47344", "unit": "" }, - "event_file_name_suffix": { "value": "4Hsize_3moderator_size_y", "unit": "" }, - "number_of_parameters": { "value": 2, "unit": "" }, - "parameters_name": { "value": "Hsize , moderator_size_y", "unit": "" }, - "event_writen_present": { "value": true, "unit": "" }, - "event_writen_file": { "value": "master_record-writen_4Hsize_3moderator_size_y.txt", "unit": "" }, - "event_writen_timestamp": { "value": "2014-01-23T19:52:38", "unit": "" }, - "event_done_present": { "value": true, "unit": "" }, - "event_done_file": { "value": "master_record-done_4Hsize_3moderator_size_y.txt", "unit": "" }, - "event_done_timestamp": { "value": "2014-01-25T00:35:55", "unit": "" }, - "event_analysis_present": { "value": true, "unit": "" }, + "sample_width": { "value": 0.015, "unit": "m" }, + "sample_height": { "value": 0.015, "unit": "m" }, + "divergence_requirement_horizontal": { "value": 0.75, "unit": "deg" }, + "divergence_requirement_vertical": { "value": 1, "unit": "deg" }, + "guide_sample_distance": { "value": 0.6, "unit": "m" }, + "lower_wavelength_limit": { "value": 1, "unit": "\u00c5" }, + "upper_wavelength_limit": { "value": 3.6, "unit": "\u00c5" }, + "moderator_width": { "value": 0.12, "unit": "m" }, + "moderator_height": { "value": 0.03, "unit": "m" }, + "moderator_sample_distance": { "value": 170, "unit": "m" }, + "parsing_variables": { "value": "guide_start , startx1 , starty1 , length1", "unit": "" }, + "parsing_min_guide_start": { "value": 2.000035881054106, "unit": "m" }, + "parsing_max_guide_start": { "value": 5.407538318585075, "unit": "m" }, + "parsing_mean_guide_start": { "value": 2.3475508029429557, "unit": "m" }, + "parsing_std_guide_start": { "value": 0.5522363822422368, "unit": "m" }, + "parsing_min_startx1": { "value": 0.006706596967962139, "unit": "m" }, + "parsing_max_startx1": { "value": 0.1460959338571846, "unit": "m" }, + "parsing_mean_startx1": { "value": 0.08885675463366878, "unit": "m" }, + "parsing_std_startx1": { "value": 0.017699812942929365, "unit": "m" }, + "parsing_min_starty1": { "value": 0.011762187831963904, "unit": "m" }, + "parsing_max_starty1": { "value": 0.14999127413576652, "unit": "m" }, + "parsing_mean_starty1": { "value": 0.13009670276273638, "unit": "m" }, + "parsing_std_starty1": { "value": 0.011522927034872269, "unit": "m" }, + "parsing_min_length1": { "value": 28.915197821153896, "unit": "" }, + "parsing_max_length1": { "value": 95.07944574028325, "unit": "" }, + "parsing_mean_length1": { "value": 64.23126877070395, "unit": "" }, + "parsing_std_length1": { "value": 10.210341803833671, "unit": "" }, + "optimization_name": { "value": "PGESKSE", "unit": "" }, + "configuration_summary": { "value": "PGESKSE", "unit": "" }, + "best_figure_of_merit": { "value": "0.25293", "unit": "" }, + "brilliance_transfer": { "value": "0.47344", "unit": "" }, + "event_file_name_suffix": { "value": "4Hsize_3moderator_size_y", "unit": "" }, + "number_of_parameters": { "value": 2, "unit": "" }, + "parameters_name": { "value": "Hsize , moderator_size_y", "unit": "" }, + "event_writen_present": { "value": true, "unit": "" }, + "event_writen_file": { "value": "master_record-writen_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_writen_timestamp": { "value": "2014-01-23T19:52:38", "unit": "" }, + "event_done_present": { "value": true, "unit": "" }, + "event_done_file": { "value": "master_record-done_4Hsize_3moderator_size_y.txt", "unit": "" }, + "event_done_timestamp": { "value": "2014-01-25T00:35:55", "unit": "" }, + "event_analysis_present": { "value": true, "unit": "" }, "event_analysis_file": { "value": "output/analysis/master_record-analyzed_4Hsize_3moderator_size_y.txt", "unit": "" }, - "event_analysis_timestamp": { "value": "2014-01-28T14:03:02", "unit": "" }, - "dataset_name": { "value": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", "unit": "" }, - "run_name": { "value": "CAMEA CAMEA31", "unit": "" }, - "scan_name": { "value": "4Hsize_3moderator_size_y", "unit": "" }, - "output_file_name_base": { "value": "PGESKSE_4Hsize_3moderator_size_y", "unit": "" }, - "dataset_access_path": { "value": "/mnt/data/simulation/CAMEA/CAMEA31", "unit": "" }, - "parameters_structure": { "value": "[{\"name\": \"Hsize\", \"value\": \"1.5\", \"index\": \"4\"}, {\"name\": \"moderator_size_y\", \"value\": \"0.03\", \"index\": \"3\"}]", "unit": "" }, - "Hsize": { "value": 4, "unit": "cm" }, + "event_analysis_timestamp": { "value": "2014-01-28T14:03:02", "unit": "" }, + "dataset_name": { "value": "CAMEA CAMEA31 Hsize 4 moderator_size_y 3 PGESKSE", "unit": "" }, + "run_name": { "value": "CAMEA CAMEA31", "unit": "" }, + "scan_name": { "value": "4Hsize_3moderator_size_y", "unit": "" }, + "output_file_name_base": { "value": "PGESKSE_4Hsize_3moderator_size_y", "unit": "" }, + "dataset_access_path": { "value": "/mnt/data/simulation/CAMEA/CAMEA31", "unit": "" }, + "parameters_structure": { "value": "[{\"name\": \"Hsize\", \"value\": \"1.5\", \"index\": \"4\"}, {\"name\": \"moderator_size_y\", \"value\": \"0.03\", \"index\": \"3\"}]", "unit": "" }, + "Hsize": { "value": 4, "unit": "cm" }, "moderator_size_y": { "value": 3, "unit": "m" } - }, + }, "techniques": [ { - "pid": "fe888574-5cc0-11ec-90c3-bf82943dec35", + "pid": "fe888574-5cc0-11ec-90c3-bf82943dec35", "name": "Simulation" } - ], - "size": 68386784, - "instrumentId": "", - "sampleId": "", + ], + "size": 68386784, + "instrumentId": "", + "sampleId": "", "proposalId": "" - }, + }, "orig_datablock": { - "size": 68386784, + "size": 68386784, "dataFileList": [ { "path": "launch_all.sh", @@ -250,4 +250,3 @@ "accessGroups": ["dmsc"] } } - \ No newline at end of file diff --git a/examples/data/published_data.json b/examples/data/published_data.json index ab2680d..54a573f 100644 --- a/examples/data/published_data.json +++ b/examples/data/published_data.json @@ -53,4 +53,4 @@ "createdAt": "2022-06-03T11:16:09.681Z", "updatedAt": "2020-09-09T09:37:58.094Z" } -] \ No newline at end of file +] diff --git a/pyscicat/client.py b/pyscicat/client.py index 1f4d5c3..1d2063a 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -152,7 +152,6 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): # err = resp.json()["error"] # raise ScicatCommError(f"Error creating Sample {err}") - def replace_dataset(self, dataset: Dataset) -> str: """ Create a new dataset or update an existing one @@ -171,7 +170,7 @@ def replace_dataset(self, dataset: Dataset) -> str: str pid of the dataset """ - + if isinstance(dataset, RawDataset): dataset_url = self._base_url + "RawDataSets/replaceOrCreate" elif isinstance(dataset, DerivedDataset): @@ -194,7 +193,6 @@ def replace_dataset(self, dataset: Dataset) -> str: """ upload_dataset = replace_dataset - def create_dataset(self, dataset: Dataset) -> str: """ Upload a new dataset. Uses the generic dataset endpoint. @@ -235,8 +233,7 @@ def create_dataset(self, dataset: Dataset) -> str: Original name, kept for for backward compatibility """ upload_new_dataset = create_dataset - - + def replace_raw_dataset(self, dataset: Dataset) -> str: """ Create a new raw dataset or update an existing one @@ -268,14 +265,12 @@ def replace_raw_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid - """ Upload a raw dataset Original name, kept for for backward compatibility """ upload_raw_dataset = replace_raw_dataset - def replace_derived_dataset(self, dataset: Dataset) -> str: """ Create a new derived dataset or update an existing one @@ -315,7 +310,6 @@ def replace_derived_dataset(self, dataset: Dataset) -> str: """ upload_derived_dataset = replace_derived_dataset - def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: """ Upsert a raw dataset @@ -348,7 +342,6 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info(f"dataset upserted {new_pid}") return new_pid - def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: """ Upsert a derived dataset @@ -382,8 +375,9 @@ def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info(f"dataset upserted {new_pid}") return new_pid - - def create_dataset_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): + def create_dataset_datablock( + self, datablock: Datablock, datasetType: str = "RawDatasets" + ): """ Create a new datablock for a dataset. The dataset can be both Raw or Derived. @@ -422,7 +416,6 @@ def create_dataset_datablock(self, datablock: Datablock, datasetType: str = "Raw """ upload_datablock = create_dataset_datablock - def create_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: """ Create a new SciCat Dataset OrigDatablock @@ -463,15 +456,11 @@ def create_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: """ upload_dataset_origdatablock = create_dataset_origdatablock - - def create_dataset_attachment( - self, - attachment: Attachment, - datasetType: str = "RawDatasets" + self, attachment: Attachment, datasetType: str = "RawDatasets" ): """ - Create a new Attachment for a dataset. + Create a new Attachment for a dataset. Note that datasetType can be provided to determine the type of dataset that this attachment is attached to. This is required for creating the url that SciCat uses. This function has been renamed. @@ -513,7 +502,6 @@ def create_dataset_attachment( """ upload_attachment = create_dataset_attachment - def find_datasets_full_query(self, skip=0, limit=25, query_fields=None): """ Gets datasets using the fullQuery mechanism of SciCat. This is @@ -561,8 +549,6 @@ def find_datasets_full_query(self, skip=0, limit=25, query_fields=None): """ get_datasets_full_query = find_datasets_full_query - - def find_datasets(self, filter_fields=None) -> List[Dataset]: """ Gets datasets using the simple fiter mechanism. This @@ -601,7 +587,6 @@ def find_datasets(self, filter_fields=None) -> List[Dataset]: """ get_datasets = find_datasets - def find_published_data(self, filter=None) -> List[PublishedData]: """ retrieve all the published data using the simple fiter mechanism. This @@ -640,7 +625,6 @@ def find_published_data(self, filter=None) -> List[PublishedData]: """ get_published_data = find_published_data - def get_dataset_by_pid(self, pid=None) -> Dataset: """Gets dataset with the pid provided. diff --git a/pyscicat/tests/test_suite_2.py b/pyscicat/tests/test_suite_2.py index 1a36106..5575b2c 100644 --- a/pyscicat/tests/test_suite_2.py +++ b/pyscicat/tests/test_suite_2.py @@ -16,9 +16,9 @@ local_url = "http://localhost:3000/api/v3/" test_dataset_files = { - 'raw' : "../../examples/data/ingestion_simulation_dataset_ess_raw_dataset.json", - 'derived' : "../../examples/data/ingestion_simulation_dataset_ess_derived_dataset.json", - 'published_data' : "../../examples/data/published_data.json" + "raw": "../../examples/data/ingestion_simulation_dataset_ess_raw_dataset.json", + "derived": "../../examples/data/ingestion_simulation_dataset_ess_derived_dataset.json", + "published_data": "../../examples/data/published_data.json", } test_datasets = {} @@ -40,7 +40,7 @@ def set_up_test_environment(mock_request): def set_up_mock_raw_dataset(mock_request): - data = test_datasets['raw'] + data = test_datasets["raw"] mock_request.post( local_url + "Datasets", @@ -61,7 +61,7 @@ def set_up_mock_raw_dataset(mock_request): def set_up_mock_derived_dataset(mock_request): - data = test_datasets['derived'] + data = test_datasets["derived"] mock_request.post( local_url + "Datasets", @@ -82,7 +82,7 @@ def set_up_mock_derived_dataset(mock_request): def set_up_mock_published_data(mock_request): - data = test_datasets['published_data'] + data = test_datasets["published_data"] mock_url = local_url + "PublishedData" print("Mock : " + mock_url) @@ -107,13 +107,10 @@ def test_scicat_ingest_raw_dataset(): scicat._token == "a_token" ), "scicat client set the token given by the server" - ownable = Ownable(**data['ownable']) + ownable = Ownable(**data["ownable"]) # Create Dataset - dataset = RawDataset( - **data["dataset"], - **ownable.dict() - ) + dataset = RawDataset(**data["dataset"], **ownable.dict()) created_dataset = scicat.create_dataset(dataset) assert created_dataset["pid"] == data["id"] @@ -123,8 +120,7 @@ def test_scicat_ingest_raw_dataset(): size=data["orig_datablock"]["size"], datasetId=created_dataset["pid"], dataFileList=[ - DataFile(**file) - for file in data["orig_datablock"]["dataFileList"] + DataFile(**file) for file in data["orig_datablock"]["dataFileList"] ], **ownable.dict() ) @@ -147,13 +143,10 @@ def test_scicat_ingest_derived_dataset(): scicat._token == "a_token" ), "scicat client set the token given by the server" - ownable = Ownable(**data['ownable']) + ownable = Ownable(**data["ownable"]) # Create Dataset - dataset = RawDataset( - **data["dataset"], - **ownable.dict() - ) + dataset = RawDataset(**data["dataset"], **ownable.dict()) created_dataset = scicat.create_dataset(dataset) assert created_dataset["pid"] == data["id"] @@ -163,8 +156,7 @@ def test_scicat_ingest_derived_dataset(): size=data["orig_datablock"]["size"], datasetId=created_dataset["pid"], dataFileList=[ - DataFile(**file) - for file in data["orig_datablock"]["dataFileList"] + DataFile(**file) for file in data["orig_datablock"]["dataFileList"] ], **ownable.dict() ) @@ -191,4 +183,3 @@ def test_scicat_find_published_data(): assert len(data) == len(returned_data) assert data == returned_data - diff --git a/requirements-hdf5.txt b/requirements-hdf5.txt index c3b2f48..d03a9f6 100644 --- a/requirements-hdf5.txt +++ b/requirements-hdf5.txt @@ -1,2 +1,2 @@ hdf5plugin -h5py \ No newline at end of file +h5py diff --git a/requirements.txt b/requirements.txt index 76aa8db..903705e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ pydantic -requests \ No newline at end of file +requests From f75027c670de23a2c47d248b7da297712d38a45a Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 14 Jun 2022 09:20:46 +0200 Subject: [PATCH 36/98] Do not log username and token --- pyscicat/client.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index d90aea4..e35dca5 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -646,7 +646,7 @@ def from_credentials(base_url: str, username: str, password: str): def get_token(base_url, username, password): """logs in using the provided username / password combination and receives token for further communication use""" - logger.info(f" Getting new token for user {username}") + logger.info(f" Getting new token") if base_url[-1] != "/": base_url = base_url + "/" response = requests.post( @@ -662,7 +662,4 @@ def get_token(base_url, username, password): raise ScicatLoginError(response.content) data = response.json() - # print("Response:", data) - token = data["id"] # not sure if semantically correct - logger.info(f" token: {token}") - return token + return data["id"] # not sure if semantically correct From b2a93e5abc0af9f2f142c5331c630b4a18bb47fe Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Tue, 14 Jun 2022 17:31:33 +0200 Subject: [PATCH 37/98] Turn into non-f-string --- pyscicat/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index e35dca5..2bc500a 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -646,7 +646,7 @@ def from_credentials(base_url: str, username: str, password: str): def get_token(base_url, username, password): """logs in using the provided username / password combination and receives token for further communication use""" - logger.info(f" Getting new token") + logger.info(" Getting new token") if base_url[-1] != "/": base_url = base_url + "/" response = requests.post( From 0c37af80dc3152bc1dd6cbbd9c2cf623720b8b82 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Wed, 15 Jun 2022 11:35:03 +0100 Subject: [PATCH 38/98] Added update dataset method with patch --- pyscicat/client.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pyscicat/client.py b/pyscicat/client.py index 2bc500a..35b78da 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -273,6 +273,39 @@ def upload_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid + def update_dataset(self, dataset: Dataset, id) -> str: + """Updates an existing dataset + + Parameters + ---------- + dataset : Dataset + Dataset to update + + id + pid (or unique identifier) of dataset being updated + + Returns + ------- + str + pid (or unique identifier) of the dataset + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + if "/" in id: + id = id.replace("/", "%2F") + dataset_url = self._base_url + "Datasets/" + id + resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True), cmd="patch") + if not resp.ok: + err = resp.json()["error"] + raise ScicatCommError(f"Error updating dataset {err}") + pid = resp.json().get("pid") + logger.info(f"dataset updated {pid}") + return pid + + # deprecated def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: """Upsert a raw dataset @@ -307,6 +340,7 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info(f"dataset upserted {new_pid}") return new_pid + # deprecated def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: """Upsert a derived dataset From 25b3e404efe55972d2bbf0d44d9b4d3da45e4d33 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Wed, 15 Jun 2022 11:43:09 +0100 Subject: [PATCH 39/98] Added update dataset mock test --- pyscicat/tests/test_client.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index b9201d1..a5d1d25 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -49,6 +49,11 @@ def add_mock_requests(mock_request): + "/RawDatasets/upsertWithWhere?where=%7B%22where%22:%7B%22sampleId%22:%20%22wowza%22%7D%7D", json={"pid": "54"}, ) + mock_request.patch( + local_url + + "/Datasets/54", + json={"pid": "54"}, + ) mock_request.post( local_url + "RawDatasets/42/origdatablocks", json={"response": "random"}, @@ -128,6 +133,11 @@ def test_scicate_ingest(): dataset_id_2 = scicat.upsert_raw_dataset(dataset, {"sampleId": "wowza"}) assert dataset_id_2 == "54" + # Update record + dataset.principalInvestigator = "B. Turtle" + dataset_id_3 = scicat.update_dataset(dataset, dataset_id_2) + assert dataset_id_3 == "54" + # Datablock with DataFiles data_file = DataFile(path="/foo/bar", size=42) data_block = Datablock( From 38d7053e53a812419d7f1a289f9d7323bd787d41 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Wed, 15 Jun 2022 14:04:38 +0100 Subject: [PATCH 40/98] Modified update dataset test --- pyscicat/tests/test_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index a5d1d25..7dd1f58 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -50,8 +50,7 @@ def add_mock_requests(mock_request): json={"pid": "54"}, ) mock_request.patch( - local_url - + "/Datasets/54", + local_url + "Datasets/54", json={"pid": "54"}, ) mock_request.post( From 98acc0cf4c5de162b4c70670848cb875578467d2 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Thu, 16 Jun 2022 08:28:53 +0100 Subject: [PATCH 41/98] Removed upsert method and tests --- pyscicat/client.py | 71 ----------------------------------- pyscicat/tests/test_client.py | 56 +++------------------------ 2 files changed, 5 insertions(+), 122 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 35b78da..fe6e840 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -305,77 +305,6 @@ def update_dataset(self, dataset: Dataset, id) -> str: logger.info(f"dataset updated {pid}") return pid - # deprecated - def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: - """Upsert a raw dataset - - Parameters - ---------- - dataset : Dataset - Dataset to load - - filter_fields - Filters to locate where to upsert dataset - - Returns - ------- - str - pid (or unique identifier) of the dataset - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - query_results = self.get_datasets(filter_fields) - if not query_results: - logger.info("Dataset does not exist already, will be inserted") - filter_fields = json.dumps(filter_fields) - raw_dataset_url = f'{self._base_url}/RawDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - resp = self._send_to_scicat(raw_dataset_url, dataset.dict(exclude_none=True)) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error upserting raw dataset {err}") - new_pid = resp.json().get("pid") - logger.info(f"dataset upserted {new_pid}") - return new_pid - - # deprecated - def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: - """Upsert a derived dataset - - Parameters - ---------- - dataset : Dataset - Dataset to upsert - - filter_fields - Filters to locate where to upsert dataset - - Returns - ------- - str - pid (or unique identifier) of the dataset - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - - query_results = self.get_datasets(filter_fields) - if not query_results: - logger.info("Dataset does not exist already, will be inserted") - filter_fields = json.dumps(filter_fields) - dataset_url = f'{self._base_url}/DerivedDatasets/upsertWithWhere?where={{"where":{filter_fields}}}' - resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error upserting derived dataset {err}") - new_pid = resp.json().get("pid") - logger.info(f"dataset upserted {new_pid}") - return new_pid - def upload_datablock(self, datablock: Datablock, datasetType: str = "RawDatasets"): """Upload a Datablock diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 7dd1f58..8d56253 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -29,29 +29,9 @@ def add_mock_requests(mock_request): ) mock_request.post(local_url + "Samples", json={"sampleId": "dataset_id"}) mock_request.post(local_url + "RawDatasets/replaceOrCreate", json={"pid": "42"}) - mock_request.get( - local_url - + "/Datasets/?filter=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", - json={"response": "random"}, - ) - mock_request.get( - local_url - + "/Datasets/?filter=%7B%22where%22:%7B%22sampleId%22:%20%22wowza%22%7D%7D", - json={"response": "random"}, - ) - mock_request.post( - local_url - + "/RawDatasets/upsertWithWhere?where=%7B%22where%22:%7B%22sampleId%22:%20%22gargleblaster%22%7D%7D", - json={"pid": "42"}, - ) - mock_request.post( - local_url - + "/RawDatasets/upsertWithWhere?where=%7B%22where%22:%7B%22sampleId%22:%20%22wowza%22%7D%7D", - json={"pid": "54"}, - ) mock_request.patch( - local_url + "Datasets/54", - json={"pid": "54"}, + local_url + "Datasets/42", + json={"pid": "42"}, ) mock_request.post( local_url + "RawDatasets/42/origdatablocks", @@ -86,7 +66,7 @@ def test_scicate_ingest(): assert size is not None # RawDataset - dataset = Dataset( + dataset = RawDataset( path="/foo/bar", size=42, owner="slartibartfast", @@ -104,38 +84,12 @@ def test_scicate_ingest(): **ownable.dict() ) dataset_id = scicat.upload_raw_dataset(dataset) - - # new dataset - dataset = RawDataset( - path="/foo/bar", - size=42, - owner="slartibartfast", - contactEmail="slartibartfast@magrathea.org", - creationLocation="magrathea", - creationTime=str(datetime.now()), - type="raw", - instrumentId="earth", - proposalId="deepthought", - dataFormat="planet", - principalInvestigator="A. Mouse", - sourceFolder="/foo/bar", - scientificMetadata={"a": "newfield"}, - sampleId="gargleblaster", - **ownable.dict() - ) - - # Update existing record - dataset_id = scicat.upsert_raw_dataset(dataset, {"sampleId": "gargleblaster"}) assert dataset_id == "42" - # Upsert non-existing record - dataset_id_2 = scicat.upsert_raw_dataset(dataset, {"sampleId": "wowza"}) - assert dataset_id_2 == "54" - # Update record dataset.principalInvestigator = "B. Turtle" - dataset_id_3 = scicat.update_dataset(dataset, dataset_id_2) - assert dataset_id_3 == "54" + dataset_id_2 = scicat.update_dataset(dataset, dataset_id) + assert dataset_id_2 == dataset_id # Datablock with DataFiles data_file = DataFile(path="/foo/bar", size=42) From d6361e0ee0b6c75cfda7ceb4731baa74c53b1ceb Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Thu, 16 Jun 2022 08:41:59 +0100 Subject: [PATCH 42/98] modified update dataset format and url encoding --- pyscicat/client.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index fe6e840..371d297 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -273,7 +273,7 @@ def upload_derived_dataset(self, dataset: Dataset) -> str: logger.info(f"new dataset created {new_pid}") return new_pid - def update_dataset(self, dataset: Dataset, id) -> str: + def update_dataset(self, dataset: Dataset, pid) -> str: """Updates an existing dataset Parameters @@ -281,7 +281,7 @@ def update_dataset(self, dataset: Dataset, id) -> str: dataset : Dataset Dataset to update - id + pid pid (or unique identifier) of dataset being updated Returns @@ -294,10 +294,15 @@ def update_dataset(self, dataset: Dataset, id) -> str: ScicatCommError Raises if a non-20x message is returned """ - if "/" in id: - id = id.replace("/", "%2F") - dataset_url = self._base_url + "Datasets/" + id - resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True), cmd="patch") + if pid: + encoded_pid = urllib.parse.quote_plus(pid) + endpoint = "Datasets/{}".format(encoded_pid) + url = self._base_url + endpoint + else: + logger.error("No pid given. You must specify a dataset pid.") + return None + + resp = self._send_to_scicat(url, dataset.dict(exclude_none=True), cmd="patch") if not resp.ok: err = resp.json()["error"] raise ScicatCommError(f"Error updating dataset {err}") From 927be04ae554b56217aab922192ea2640ee488b6 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Thu, 16 Jun 2022 08:46:08 +0100 Subject: [PATCH 43/98] flake8 linting changes --- pyscicat/tests/test_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 8d56253..32af33d 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -14,7 +14,6 @@ Attachment, Datablock, DataFile, - Dataset, RawDataset, Ownable, ) From 08d524afd958935940e2ceafa505ba695ebdf573 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Thu, 16 Jun 2022 17:07:45 +0200 Subject: [PATCH 44/98] Changed functions names according to what agreed on PR #20 --- pyscicat/client.py | 149 +++++++++++++++++++++++++++++++++------------ 1 file changed, 109 insertions(+), 40 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 1d2063a..7a7d886 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -152,12 +152,13 @@ def _send_to_scicat(self, url, dataDict=None, cmd="post"): # err = resp.json()["error"] # raise ScicatCommError(f"Error creating Sample {err}") - def replace_dataset(self, dataset: Dataset) -> str: + def datasets_replace(self, dataset: Dataset) -> str: """ Create a new dataset or update an existing one This function was renamed. It is still accessible with the original name for backward compatibility - The original name was upload_dataset + The original names were upload_dataset replace_datasets + This function is obsolete and it will be remove in next relesases Parameters @@ -191,15 +192,17 @@ def replace_dataset(self, dataset: Dataset) -> str: Upload or create a new dataset Original name, kept for for backward compatibility """ - upload_dataset = replace_dataset + upload_dataset = datasets_replace + replace_dataset = datasets_replace - def create_dataset(self, dataset: Dataset) -> str: + + def datasets_create(self, dataset: Dataset) -> str: """ Upload a new dataset. Uses the generic dataset endpoint. Relies on the endpoint to sense the dataset type This function was renamed. It is still accessible with the original name for backward compatibility - The original name was upload_new_dataset + The original name were create_dataset and upload_new_dataset Parameters ---------- @@ -232,14 +235,17 @@ def create_dataset(self, dataset: Dataset) -> str: Upload a new dataset Original name, kept for for backward compatibility """ - upload_new_dataset = create_dataset + upload_new_dataset = datasets_create + create_dataset = datasets_create + - def replace_raw_dataset(self, dataset: Dataset) -> str: + def datasets_raw_replace(self, dataset: Dataset) -> str: """ Create a new raw dataset or update an existing one This function was renamed. It is still accessible with the original name for backward compatibility - The original name was upload_raw_dataset + The original names were repalce_raw_dataset and upload_raw_dataset + THis function is obsolete and it will be removed in future releases Parameters ---------- @@ -269,14 +275,17 @@ def replace_raw_dataset(self, dataset: Dataset) -> str: Upload a raw dataset Original name, kept for for backward compatibility """ - upload_raw_dataset = replace_raw_dataset + upload_raw_dataset = datasets_raw_replace + replace_raw_dataset = datasets_raw_replace - def replace_derived_dataset(self, dataset: Dataset) -> str: + + def datasets_derived_replace(self, dataset: Dataset) -> str: """ Create a new derived dataset or update an existing one This function was renamed. It is still accessible with the original name for backward compatibility - The original name was upload_derived_dataset + The original names were replace_derived_dataset and upload_derived_dataset + Parameters ---------- @@ -308,11 +317,14 @@ def replace_derived_dataset(self, dataset: Dataset) -> str: Upload a derived dataset Original name, kept for for backward compatibility """ - upload_derived_dataset = replace_derived_dataset + upload_derived_dataset = datasets_derived_replace + replace_derived_dataset = datasets_derived_replace - def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: + + def datasets_raw_upsert(self, dataset: Dataset, filter_fields) -> str: """ - Upsert a raw dataset + Update or insert a raw dataset + This function is obsolete is going to be removed in future releases Parameters ---------- @@ -342,9 +354,13 @@ def upsert_raw_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info(f"dataset upserted {new_pid}") return new_pid - def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: + upsert_raw_dataset = datasets_raw_upsert + + + def datasets_derived_upsert(self, dataset: Dataset, filter_fields) -> str: """ - Upsert a derived dataset + Update or insert a derived dataset + This function is obsolete and is going to be removed in future releases Parameters ---------- @@ -375,14 +391,19 @@ def upsert_derived_dataset(self, dataset: Dataset, filter_fields) -> str: logger.info(f"dataset upserted {new_pid}") return new_pid - def create_dataset_datablock( + upsert_derived_datasets = datasets_derived_upsert + + + def datasets_datablock_create( self, datablock: Datablock, datasetType: str = "RawDatasets" ): """ Create a new datablock for a dataset. The dataset can be both Raw or Derived. It is still accessible with the original name for backward compatibility - The original name was upload_datablock + The original names were create_dataset_datablock and upload_datablock + This function is obsolete and will be removed in future releases + Function datasets_origdatablock_create should be used. Parameters ---------- @@ -414,14 +435,16 @@ def create_dataset_datablock( Upload a Datablock Original name, kept for for backward compatibility """ - upload_datablock = create_dataset_datablock + upload_datablock = datasets_datablock_create + create_dataset_datablock = datasets_datablock_create - def create_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: + + def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: """ Create a new SciCat Dataset OrigDatablock This function has been renamed. It is still accessible with the original name for backward compatibility - The original name was upload_dataset_origdatablock + The original names were create_dataset_origdatablock and upload_dataset_origdatablock Parameters ---------- @@ -454,9 +477,11 @@ def create_dataset_origdatablock(self, origdatablock: OrigDatablock) -> dict: Create a new SciCat Dataset OrigDatablock Original name, kept for for backward compatibility """ - upload_dataset_origdatablock = create_dataset_origdatablock + upload_dataset_origdatablock = datasets_origdatablock_create + create_dataset_origdatablock = datasets_origdatablock_create + - def create_dataset_attachment( + def datasets_attachment_create( self, attachment: Attachment, datasetType: str = "RawDatasets" ): """ @@ -465,7 +490,7 @@ def create_dataset_attachment( that this attachment is attached to. This is required for creating the url that SciCat uses. This function has been renamed. It is still accessible with the original name for backward compatibility - The original name was upload_attachment + The original names were create_dataset_attachment and upload_attachment Parameters ---------- @@ -500,9 +525,11 @@ def create_dataset_attachment( Create a new attachement for a dataset Original name, kept for for backward compatibility """ - upload_attachment = create_dataset_attachment + upload_attachment = datasets_attachment_create + create_dataset_attachment = datasets_attachment_create - def find_datasets_full_query(self, skip=0, limit=25, query_fields=None): + + def datasets_find(self, skip=0, limit=25, query_fields=None): """ Gets datasets using the fullQuery mechanism of SciCat. This is appropriate for cases where might want paging and cases where you want to perform @@ -516,7 +543,7 @@ def find_datasets_full_query(self, skip=0, limit=25, query_fields=None): This function was renamed. It is still accessible with the original name for backward compatibility - The original name was get_datasets_full_query + The original name was find_datasets_full_query and get_datasets_full_query Parameters ---------- @@ -547,13 +574,17 @@ def find_datasets_full_query(self, skip=0, limit=25, query_fields=None): find a set of datasets according the full query provided Original name, kept for for backward compatibility """ - get_datasets_full_query = find_datasets_full_query + get_datasets_full_query = datasets_find + find_datasets_full_query = datasets_find + - def find_datasets(self, filter_fields=None) -> List[Dataset]: + def datasets_get_many(self, filter_fields=None) -> List[Dataset]: """ Gets datasets using the simple fiter mechanism. This is appropriate when you do not require paging or text search, but want to be able to limit results based on items in the Dataset object. + This function has been renamed and the old name has been mantained for backward compatibility + The previous names are find_datasets and get_datasets For example, a search for Datasets of a given proposalId would have ```python @@ -585,13 +616,17 @@ def find_datasets(self, filter_fields=None) -> List[Dataset]: find a set of datasets according to the simple filter provided Original name, kept for for backward compatibility """ - get_datasets = find_datasets + get_datasets = datasets_get_many + find_datasets = datasets_get_many + - def find_published_data(self, filter=None) -> List[PublishedData]: + def published_data_get_many(self, filter=None) -> List[PublishedData]: """ retrieve all the published data using the simple fiter mechanism. This is appropriate when you do not require paging or text search, but want to be able to limit results based on items in the Dataset object. + This function has been renamed and the old name has been maintained for backward compatibility + The previous name are find_published_data and get_published_data For example, a search for published data of a given doi would have ```python @@ -623,10 +658,15 @@ def find_published_data(self, filter=None) -> List[PublishedData]: find a set of published data according to the simple filter provided Original name, kept for for backward compatibility """ - get_published_data = find_published_data + get_published_data = published_data_get_many + find_published_data = published_data_get_many - def get_dataset_by_pid(self, pid=None) -> Dataset: - """Gets dataset with the pid provided. + + def datasets_get_one(self, pid=None) -> Dataset: + """ + Gets dataset with the pid provided. + This function has been renamed. Provious name has been maintained for backward compatibility. + Previous names was get_dataset_by_pid Parameters ---------- @@ -643,6 +683,9 @@ def get_dataset_by_pid(self, pid=None) -> Dataset: return None return response.json() + get_dataset_by_pid = datasets_get_one + + # this method is future, needs testing. # def update_dataset(self, pid, fields: Dict): # response = self._send_to_scicat( @@ -654,11 +697,14 @@ def get_dataset_by_pid(self, pid=None) -> Dataset: # return None # return response.json() - def get_instrument(self, pid: str = None, name: str = None) -> dict: + def instruments_get_one(self, pid: str = None, name: str = None) -> dict: """ Get an instrument by pid or by name. If pid is provided it takes priority over name. + This function has been renamed. Previous name has been maintained for backward compatibility. + Previous name was get_instrument + Parameters ---------- pid : str @@ -694,9 +740,15 @@ def get_instrument(self, pid: str = None, name: str = None) -> dict: return None return response.json() - def get_sample(self, pid: str = None) -> dict: + get_instrument = instruments_get_one + + + def samples_get_one(self, pid: str = None) -> dict: """ - Get an sample by pid. + Get a sample by pid. + This function has been renamed. Previous name has been maintained for backward compatibility. + Previous name was get_sample + Parameters ---------- @@ -719,9 +771,14 @@ def get_sample(self, pid: str = None) -> dict: return None return response.json() - def get_proposal(self, pid: str = None) -> dict: + get_sample = samples_get_one + + + def proposals_get_one(self, pid: str = None) -> dict: """ Get proposal by pid. + This function has been renamed. Previous name has been maintained for backward compatibility. + Previous name was get_proposal Parameters ---------- @@ -743,9 +800,14 @@ def get_proposal(self, pid: str = None) -> dict: return None return response.json() - def get_dataset_origdatablocks(self, pid: str = None) -> dict: + get_proposal = proposals_get_one + + + def datasets_origdatablocks_get_one(self, pid: str = None) -> dict: """ Get dataset orig datablocks by dataset pid. + This function has been renamed. Previous name has been maintained for backward compatibility. + Previous name was get_dataset_origdatablocks Parameters ---------- @@ -767,9 +829,14 @@ def get_dataset_origdatablocks(self, pid: str = None) -> dict: return None return response.json() - def delete_dataset(self, pid: str = None) -> dict: + get_dataset_origdatablocks = datasets_origdatablocks_get_one + + + def datasets_delete(self, pid: str = None) -> dict: """ Delete dataset by pid + This function has been renamed. Previous name has been maintained for backward compatibility. + Previous name was delete_dataset Parameters ---------- @@ -792,6 +859,8 @@ def delete_dataset(self, pid: str = None) -> dict: return None return response.json() + delete_dataset = datasets_delete + def get_file_size(pathobj): filesize = pathobj.lstat().st_size From 610440ac1eb918f6446c432e2518a338884d4154 Mon Sep 17 00:00:00 2001 From: Massimiliano Novelli Date: Fri, 17 Jun 2022 09:29:06 +0200 Subject: [PATCH 45/98] Implemented new naming --- pyscicat/client.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 7a7d886..946b649 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -195,7 +195,6 @@ def datasets_replace(self, dataset: Dataset) -> str: upload_dataset = datasets_replace replace_dataset = datasets_replace - def datasets_create(self, dataset: Dataset) -> str: """ Upload a new dataset. Uses the generic dataset endpoint. @@ -238,7 +237,6 @@ def datasets_create(self, dataset: Dataset) -> str: upload_new_dataset = datasets_create create_dataset = datasets_create - def datasets_raw_replace(self, dataset: Dataset) -> str: """ Create a new raw dataset or update an existing one @@ -278,7 +276,6 @@ def datasets_raw_replace(self, dataset: Dataset) -> str: upload_raw_dataset = datasets_raw_replace replace_raw_dataset = datasets_raw_replace - def datasets_derived_replace(self, dataset: Dataset) -> str: """ Create a new derived dataset or update an existing one @@ -320,7 +317,6 @@ def datasets_derived_replace(self, dataset: Dataset) -> str: upload_derived_dataset = datasets_derived_replace replace_derived_dataset = datasets_derived_replace - def datasets_raw_upsert(self, dataset: Dataset, filter_fields) -> str: """ Update or insert a raw dataset @@ -356,7 +352,6 @@ def datasets_raw_upsert(self, dataset: Dataset, filter_fields) -> str: upsert_raw_dataset = datasets_raw_upsert - def datasets_derived_upsert(self, dataset: Dataset, filter_fields) -> str: """ Update or insert a derived dataset @@ -393,7 +388,6 @@ def datasets_derived_upsert(self, dataset: Dataset, filter_fields) -> str: upsert_derived_datasets = datasets_derived_upsert - def datasets_datablock_create( self, datablock: Datablock, datasetType: str = "RawDatasets" ): @@ -438,7 +432,6 @@ def datasets_datablock_create( upload_datablock = datasets_datablock_create create_dataset_datablock = datasets_datablock_create - def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: """ Create a new SciCat Dataset OrigDatablock @@ -480,7 +473,6 @@ def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: upload_dataset_origdatablock = datasets_origdatablock_create create_dataset_origdatablock = datasets_origdatablock_create - def datasets_attachment_create( self, attachment: Attachment, datasetType: str = "RawDatasets" ): @@ -528,7 +520,6 @@ def datasets_attachment_create( upload_attachment = datasets_attachment_create create_dataset_attachment = datasets_attachment_create - def datasets_find(self, skip=0, limit=25, query_fields=None): """ Gets datasets using the fullQuery mechanism of SciCat. This is @@ -577,7 +568,6 @@ def datasets_find(self, skip=0, limit=25, query_fields=None): get_datasets_full_query = datasets_find find_datasets_full_query = datasets_find - def datasets_get_many(self, filter_fields=None) -> List[Dataset]: """ Gets datasets using the simple fiter mechanism. This @@ -619,7 +609,6 @@ def datasets_get_many(self, filter_fields=None) -> List[Dataset]: get_datasets = datasets_get_many find_datasets = datasets_get_many - def published_data_get_many(self, filter=None) -> List[PublishedData]: """ retrieve all the published data using the simple fiter mechanism. This @@ -661,7 +650,6 @@ def published_data_get_many(self, filter=None) -> List[PublishedData]: get_published_data = published_data_get_many find_published_data = published_data_get_many - def datasets_get_one(self, pid=None) -> Dataset: """ Gets dataset with the pid provided. @@ -685,7 +673,6 @@ def datasets_get_one(self, pid=None) -> Dataset: get_dataset_by_pid = datasets_get_one - # this method is future, needs testing. # def update_dataset(self, pid, fields: Dict): # response = self._send_to_scicat( @@ -742,13 +729,12 @@ def instruments_get_one(self, pid: str = None, name: str = None) -> dict: get_instrument = instruments_get_one - def samples_get_one(self, pid: str = None) -> dict: """ Get a sample by pid. This function has been renamed. Previous name has been maintained for backward compatibility. Previous name was get_sample - + Parameters ---------- @@ -773,7 +759,6 @@ def samples_get_one(self, pid: str = None) -> dict: get_sample = samples_get_one - def proposals_get_one(self, pid: str = None) -> dict: """ Get proposal by pid. @@ -802,7 +787,6 @@ def proposals_get_one(self, pid: str = None) -> dict: get_proposal = proposals_get_one - def datasets_origdatablocks_get_one(self, pid: str = None) -> dict: """ Get dataset orig datablocks by dataset pid. @@ -831,7 +815,6 @@ def datasets_origdatablocks_get_one(self, pid: str = None) -> dict: get_dataset_origdatablocks = datasets_origdatablocks_get_one - def datasets_delete(self, pid: str = None) -> dict: """ Delete dataset by pid From 1ed275739dec9220ea11afe570b0dab019e3620a Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 30 Jun 2022 10:17:41 +0200 Subject: [PATCH 46/98] Add more model fields --- pyscicat/model.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pyscicat/model.py b/pyscicat/model.py index 6de34c4..a760906 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -145,9 +145,7 @@ class RawDataset(Dataset): principalInvestigator: Optional[str] creationLocation: Optional[str] dataFormat: str - type: DatasetType = "raw" - createdAt: Optional[str] # datetime - updatedAt: Optional[str] # datetime + type: DatasetType = DatasetType.raw dataFormat: Optional[str] endTime: Optional[str] # datetime sampleId: Optional[str] @@ -160,12 +158,13 @@ class DerivedDataset(Dataset): Derived datasets which have been generated based on one or more raw datasets """ - investigator: Optional[str] + investigator: str inputDatasets: List[str] - usedSoftware: List[str] # not optional! + usedSoftware: List[str] jobParameters: Optional[dict] jobLogData: Optional[str] scientificMetadata: Optional[Dict] + type: DatasetType = DatasetType.derived class DataFile(MongoQueryable): @@ -178,12 +177,13 @@ class DataFile(MongoQueryable): path: str size: int time: Optional[str] + chk: Optional[str] uid: Optional[str] = None gid: Optional[str] = None perm: Optional[str] = None -class Datablock(Ownable): +class Datablock(Ownable, MongoQueryable): """ A Datablock maps between a Dataset and contains DataFiles """ @@ -194,18 +194,20 @@ class Datablock(Ownable): packedSize: Optional[int] chkAlg: Optional[int] version: str = None + instrumentGroup: Optional[str] dataFileList: List[DataFile] datasetId: str -class OrigDatablock(Ownable): +class OrigDatablock(Ownable, MongoQueryable): """ - A Original Datablock maps between a Dataset and contains DataFiles + An Original Datablock maps between a Dataset and contains DataFiles """ id: Optional[str] # archiveId: str = None listed in catamel model, but comes back invalid? size: int + instrumentGroup: Optional[str] dataFileList: List[DataFile] datasetId: str From a4aee32d1116d1b3cf5811cd5563f74f2d9a967a Mon Sep 17 00:00:00 2001 From: Tobias Richter Date: Tue, 5 Jul 2022 16:50:23 +0200 Subject: [PATCH 47/98] add logo design by Max approved by PSF --- docs/pyscicatlogo.png | Bin 0 -> 54350 bytes docs/pyscicatlogo.svg | 164 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 docs/pyscicatlogo.png create mode 100644 docs/pyscicatlogo.svg diff --git a/docs/pyscicatlogo.png b/docs/pyscicatlogo.png new file mode 100644 index 0000000000000000000000000000000000000000..d6b31afbeb10de1bad64f0c3e165acbb628a9964 GIT binary patch literal 54350 zcmd42V}D$K+dVuJ+qP||aib=UZL_hh#*J;;wkBqiG>vWB&Yk|w>;E3^2lJUd`@zhy zzlYaa6Rsd9i3o=W2LJ#NrM`(N0RX^i004{%2I}(;{JU-a=Nqj3H%%u10IC1q3pi|7 zYVvs#*I8V{S=r9a+0D?=6yWCO#$;)0_AVn_{&~EML~!H zZN6>#8onKz;fP*SK(avduq0hgM94`@)R>RnbvMa6~p9=*QD zO4;Ae&*!5i%P%j{3Z+w>XPokTznS7asch&LSIvRjygC#<)-Yuv^P6oxv&-~4Z>w2r zvD?@RpR4>l45>so9aTB6r5-;&r_0S=JE%(nzy_4Ti;7Leh`H)r4e4+FWfqju@HJaa(h#;XnGt6!-S>VApllyyB}McApOq%$9lL27H(v zOlApn*llz_Nu8g_QDT3*LN`wcUX?*lIN$HZ-V4GH`eaC)S5)bd{Ok~M;Kp1fEO!F} zfYX|k!NR$>*l=Nh<&QHY+FUr)0IfIc1+6e&EV>V}pHpCuk#z5xjc;NV*Wt~&-bN9V zf=}7_Odl;#S!r~`^#ANWUF&$5QoY+ow=lXJi6ejT`1Hxs(rBVqiH^r!QjG1&{T!g% zoBTy}t_R}s6dW4q^-kix19x|u`1y&k;pXB>wL$ZysrCiiaWg8l*nWR^74y1&=zX9pYH6pMk8_O=i~c5?fb}g z`;_yRvqhJ?^iC+EdR)6K=sq-u&uXQdE;SrT1BAkrQ}~@VR6lkC zz@csV#ouyJuU&-h+r6y&o9!(A2zpXAd{2QZ};-oe5 z{$}CpcW2X~vSNq&huS*HMjtxYa$5NJ>cdcviJVcd5d760OdpA{OKQ}e*l3Y+BPU*D zY(6j-28i=)tT+`T-G~oZ38jLhSMVwQO0>+;Kn$g&Q~$tiXIuCr+Pf|L;4?7Rq389q z7lEESPd7~Z>Cv)A8!x%PGS}(r(X@EH>9@IeLV#D0#BGb5$Ngj$-^EiOcK0Qp-qV=+ zx{Up_r! zC%S4G!jsNwFEixip>Ucp)7rRam8;_AXX0=ke@blAKW?fse0OBov)-1ysy`-${zf*a z%EkY4=nThil{7c}>-gvMwT_zJBS54frO0$aUy&leq(c+J<(4FG$w>NFlyyz@gV4z`T_*2T(4R#GyAu) zGeCBS#zeN$X!gRZR)gvN%)|Swo1alsTA5>{(gWRUCmZI@7t4o{2>sXX8$c%(s5xP7 zc;Gsxy7PL>j%Q98l5?m4eq{#+T<)-*FleSanMXhMMQ5NlDF|72cTes+uijJi-jZ9mXI|lQ-N2{K?=%mq(Z@mTP!wpjvC1Qqz-0TZd_s5Kqh5tx z5O;SrW;lNDFS4ToE1^(4!p%xQIOO{Rm#!|dno!mPHqbBb972pYJ6hefDvQhQq!rr^ z(@)FVPH9q##=aDSsV8xYi^)lsUt|k-brXS{^ZRa6LfQV&r8 zD01!&yS04!ZC@_4`7LPb3)8ak^i7N@gKF+HbKvF!)7s4mxA!2t-@QyYTV}FSK*d;k zetClVjFdSc(&j|hho1XhlDgof3cW*T{mF3)zq;xMOs^bY@sQxJ5dvz~LTlmD1hgG< z3AIGPH15&}ULw#x82#yuUYR0Q>u?j6e89dP;j02g4U8!hya-zn{d{-6TxCf>3}imo z2q8u632vHUL3#I~t@h-{-{H>Ph4U!7VM_Dzt(Omcyy&WNEfC0w!@sN`r5sg~$>#8wo$Gl)P<@ksC1Vjb2+Wj0Wm`?kqNIMqys-3u@BPKo zX)!_lHFl5*3rg^-iN$oNgcsH^I+zp~kE$~>H~TsUZcI5N%pZiZvbgRR;9KFP0Ry^3 zSGT}V-oHEN@L3H&?|$9k-(s?wv@*W&{V&ru5p$_3WB@Z`offF7vd}q0uCWq z;@pS5{(A}=?cYUXvuD?`E+ra%%RJSWb5i+it+DCTg>xbvKF3Gw5JYv{1UQ`xmFk}N z=az^OWnd>FXc*uf{Vt?}AtZM2P6XPx|MEhvA? zDSxH{T%iB)V>fWD7L6QFRedEkFkSNh(<8Aa1{Klh3h$;hFu8t_|AlC-p*d)gxF_;d zQ2uI-b&Q%hiGA}f+Qw#TWfQY?TA=7c`Bl|_+^y?yZb~AlK>AIfnhu}pGhw^rR3Bvi zB!{iquemIo31+wd3kuPd<}O%JGOWr|)k#gydk3|Jx#Q;Wb3f@mHD!L$2k5cK(IJ1%oTGd3d_>P;rGi0(e%#&NY8VLPGrxVZ%8YrP zVD!B0SB}-YKGI9pG=5?M4!eEMy!?(WFML1bh0dVGmuuAKdSV3|hYP9Bn8W-P)&<9$ zciIUx<4G8YAZR1%E(TrVUolbRO)%<|-zp2&*iQY5+T!!0u}V#Ris$MfmNT=<#5;W2 zoEmu@IX^S(b6>Mgm4VUuaNh(4^n8hfN-utj3U_|G4qr~wOW)^X;7@))W57|6=5DzG z6SmFyeM!9jlCa@=_WOlNU)}Y{%lG(**e05Rs9>5~%uPCV^Ad>EmZZ&@By2v42|i@J zlVCA*JiU$0WBg|!S0-U3AEx%^MkcSZw7-kiC&I5y@25IkiWXV;m%~vfVvj-QobHD? zOOpSr3LBvSQIdVUh^_FvoHlHfmAA;}PV&AyTnGs?IBD~b3hoP#A{4rRFFtSWDk);b z05znlJtlB^2b?DXZctN-$Ur1V{1BSxftLUF&C=jHrSWw94`#Sz){E?UW z?zAmQ)bo}ATr>6x?v|4J3+^pF^vg6Eaz-)Cfh+=`Ep9#~nN;+zXN81^$vF0u9dBe> z;J@6T@iCBHWPQyEghpOB{pDJmShMl54^Q$;nViW25jHY&A}%_8Dt9Fq_Ewj@g^-cA8zva7VZEG)#tlW<$HsBEJ2ve!TAoZ6 zKxM91{9dKwjYtUXulFvI>N{op@Nsl>gpvjIpGBiVd42M7vy&8MS2io_R{zYHQafco_LX?T{VX`Vw}K;s-+4f ze2azMNdQN!qlQl1)&Q&8<_`s=nNm6mu`O1?B-1PN61w7Y>3u33y^@O;aEM<~* zE`pQJBvu@N*0SlvsH7K$E(lU_M4O|{a{z3CiV?NO2h3Z5%a>GfqqJcyi#_q6PGtc-RJ87}2)L`}=8-rm+#cM+GJ0z^& z)mWEvnt^mw|4Ud*y3Ah=Z)+QGYs0Lo;=As<>n&@pD>qaY=4_gcBKVOmSIOU-ZIn+h ziMIO9R_>={$|9PdEww)f#mhGY%3So3JhqwyZKu-BHO@|z-sV<4btyIrk5*e=!N+-} z#_A!R@o|3zsyZ%;So((JCsP}}1?9tN-H$8mi#C-yzPf*okNY=x^NaJ<_?%f`mw$mh zL5oY#a~Q14&rXFDTVRU5%-^mF_Sjpe3mQjYvsn$)RP-zx;o{ zJcqYn^vm_R`F%l5_RKncKT2KKWeyU($?a>sMy0C=W0daV8zEy*os2bP)|TBLx##iD zpG%O%<0M|woxyaVybczr=YMN-8MjdS)p<3$W{r zIIcvwt8+rd!s(c==@XOgZqFET3>`)HPi_NZ5ZY?Zgknhn|I4?}haVdo^ZtxCgHLdx z2iib-$DSC~<6*ta%kS zj=t-VFm_}$D=q1_<$7sw7`8Jh%p3ipp7~14zAHhqASgkk#HY+RPH2*LBPg8y@7fxQ zVfL*?SnULYj>Cy^%f)feX0Az5iDj;nsR*t2XDe>*pNNAUYMi;;9F@O>YEO%0`o-Gk zi3t6}a01JpaRJnYEn%H^?I+cITp^42`WpeOjEtLQN85Y}_+G82f0&=LLc3q6d)|Wj zOMhE858vV(7c2NKjNY-3GZ$M_Ptx@I`G7Ds=z+E-QS^F0u>5^VJ*;CSk=mX+lP|3H z5(f`U9#6_ian!V8AZm?YqT|9V{?$l{F_lr=1fV)|4~&8Ircql>`MTS$5S2Uq+R5qS zMs}A>3}KE)3*RGdVREI;BF#q6Dwyn2@E?u79+~<0%p7O&_I!MgI+pm zd8)Y5h?r`h1Hq4|3d{7-cy_>Nvjz79+ssmK_FxkeYp!lN?G(u0|NU1!k>_dik|Y#; z5D1*L-?Q9>C%w_8!*+AIzJ5^CzaJ_ARYM8Nrb&#n_PpbCLKMR zPRM`ct)09v1d9;757$jin@!`YEi*nv9*^<6V*7g_f)cN__j>Ygd4_Cis_6t3r{91-+J@tg74gZ`xTM!pmm1 zsNGtTKF3h%&w1}nKvm$Z*^bRJ-E6Jw&rP$c-rwsrdzbkwAmz$4~`}!RSt3X?%@w z9;6gtl`!_1bUliyep0GEVq%k86{nv7Ql0q}QLdM@ATofAO?~b&_*efylUHItvYI2| z0qMmalSXxqB>sP|i{w>xA0m1q)S4_g0x%V7BiH@YMb1W~C`3`(+&%e*6WqEU5o^*! z)@jk5(5DwpKC+P1hZ%EdI0rzr#}tpX}(qX#=m98=bIWKnBl9Qp4IhRhKs^7 zHJ=Q`X93S+T$=;iMpt9>lmJcSE-US}l_r%N|F z06QMsGkKB|GaY(*|7Hx%1~C|cXA!2|h!@svR|aD`4~aq{ffjKSxOO zEGM@TzZJ`7+?gs51#*!_Q8?i(h60jtY{dw{6|5E6Kk3^K|CIV;!n9u!f00?p0OSA+ zaE1s7cq{EvhN!78MJ?VgCa=Zuvi@iWgOwy`tTb5)Sy|yN{Z>Ih$40NN-}teMX$jp# z!Zz9CTk5#dkkztuWRA;Stvsq~TcgSHKF7*a;FsDZS}{t|lq!oU)EC+7b6-=HzoQs9 z_KnO`^`M~FUa$M!4(9gdNf>))i;sM%H!MNP1PTX)UB;5t$hVyj3{(P zr+$sQbX(Q9fT5P~%Wjtol)Hh1^K-m6%JV`Hx?Ob8dfNvbk^e*KGK(Xv&^FV5n^Dh^ zes*X54xUfU@B!%lGJ1NTv8OS!Riya`tq~yHL^)m0SB4Xvta z=KKeZr-Lm}JZP^pQwzj9?nH4{f3E4ys&97*YYVI+?V`amM5OrR>pc+Qn0lt^DP zzZU8V37Q*>8_sEf7wAhij~J&LH*J{<0CJ1XBmeV)?BC32kc$KhQv$wi>(=xq4&N;) zRt>>&^nGfW_A^$q#PD2e{oiHMtT#<0oUS24Yx9=|8XeIj8}+L%#7vn)m8vx{ZM?1r z(<6j44QBF!La$I0E=_=3=yw10m`Z(zT>Da~{3#><^t?BicnvEMR%-}_i7&qeHJ*Q3L_pB85v=(4cl25dxe#PdS>N*No zoTqZ55CvJ$GE}%u2#N-ztMbtO%LA1mm+Rd2rAB6>!G(wso{v&E%r13SS$^wsUh<^T9b{RNKSg8(Lo%c9$lp^w++;~zac zVrJrIisKv}z-@Dhk;{Wh9&qa07u*mR*ZnPouX#U&F^4=_&p(}nBVjIq#v@I3xF<#u zjnJ#dAU0Ct$P1dmORWfpIq=92t)n?}>r*)YqO>O4i{q-GXWS_`&`Q9{0$BwP7K*f> z>pENrwm_BCY{SjCvwXN4c{kGZS>RDXe9s@cVq$uiKe1PR#*-Njz*{TAGMUK^Dvo&p zYGNiOt{wXq7;#fEO;vMdLem4nw(I1;M1_&NFTi)RH03-}Gr(dY7 zBGtnP6jgAvnvTD)l%UrI^}T%`82;d1G7Y@UI(Qq4I8-f>SbpL=O1@rj8 zQ{%bGg-!`wZIepXU6OXO*)SlW>ZuNyXOI&KhK=426O2bD{Ww%IZCgF!-{gzPbeY$b z+`lRF+jJ0Q;o#Q>OR(M`lb1|0m|kYT9`?)gcZ58LhlPWaA?ZJaXE$e&x1=&>{?O3RNsik*#A z0~Ch(^0j6KQf)Xr-373$HJc)+~VNcEsv;cSmFx;h}@nG&5 zLSE50MPzo{8o}k%e;wyvedz}<3C?+DQj`HSAhVvdKIy^AvKI+|6$)9C=A?qSG9szY zPQu_W%*oYLk+6-w_5LIcRg1sHa?vh60eJ0lU}?RfW4Rtdbv>_X_ohN?)u?7)Z~!!= z{UToC+%yl!O1F;akkRXDqTR?1X^wqNh| z)CbfS*k-YQ)~ru3j$oXs&PqylvUEnOWhM^H*XIj~xv)fPY&{)xT2x(V0rdSZ?0Vka zk=fs%s01=zT#99ym&v2c@IRVqzqAXK7CyhDPz6-RmNxNHDK<_L`P&W^U=|^PB~VMsr&$U+67ZeUG*pN{i(cvR&+r;z!x?ge zjLZZrdSebL_myeL)vqMzIml_QP?r9KX`)Y{elO>Byq#%)Huh8Lz@#XN^&aYK6BTbwyS=Jb2Q-gT7+TsTZ(-t+OkKC<+)@|THV`XREOH830w1F&WTt*yL z@0Fut^P!?wFVmXpR$NY?^TH!kuZ2n+5W(<^aENwGW(z zedNoF_4nQa-Gr$4|4cY{_P^m2dTzdPt8PTLYU=ui{aiFN(goxtF4qPKAjFAUpo+Zs z7HD(Gdb@^uAE^a|lm%o-JgqHTOy?6;BQ`F(Y<%133Zb+3!-2Rwf6Te22#HrWl?5id zVONpYlFjQl6LxrA>eS=?LWxs=eP!)X<8rtmT7yeb*Y|n#GGIwpibBK=S=Jy?z zaFdIV2co0~HQ&MysCd1b>NTzgsv|(eg6HflHx>k72GsBLBUc#RY}i}(JG?*o0x9j# zl7A%=YgOg&CD909lGtG^Ga{T%R-NM0`Rf<#wt27uBWtVV@ZRV<2ULg!!xhb!tF3Wg z)ES+?(ARaLYWZHvqE8+&Dp58g^J)aUXREJR)1!CIZHygr@rb!uE5J9azei)NVR!au zee@#BbeUQ{-)u2yM0iI4xKpHc#TAMMCD>z?$V&B$N)d?FNV0suWrBk>9+Gf-@Z9*G zc#lQ!CnxP!l`$lRxq_oA1wb$qdTUZ7JIuFB3cU9ElAQ)$Q|k147^G)c-NxZ@(>WoA z>R@yeQQA}HA;VPOfd5S}LE1M*Bj;WI4Su9XPb?DVpkt@YIO@~k-v?{{B;=3M7 zm`fU@I4*Dg9lkSj*AM*>1dp^kCwbx|TqB*d+?tYygy8ESvW zHPhhm>hCI`hR{Mf6dFL|7HDq$%`-*DtyVRG%^nXODV7B*16>fI_Ss7kBh)$SG%^V@ zS9iCKUZaHA>4)?}K`WE*RHTkyj!dDax#$&}j{6Dw)z+{OzW!aPUz5(#|G7iTzrs?Q z+i7R-kI?%)baVTorRDOt09rD?D;R*=sBSZ7!R+T1N?e^;`$l)XceRc@?HLMt_Cz`u z;CE^?YXeJ-*+-wnpqG>!B0Qr6l*nGFoZlg?rvEA#JMRm9xM9G-c#UXhq;;NHkDSi% zK|V#;cEXr8+a979W?I!U0XzQ5UF<~A5>O!nU!+IC2#8m}rr#I#%a}4_d%WoT@m)bU ztkHD+5CH-)T-?KU@~{v#_WM@FLf#|n=cY=BPg_D@FQ-D(@gd(r{KauaK5QEfrnydl zMxO+ednA||`Tmgg8%7w5y7>TDEll=d5)b>(4}FahSovr!kL^mybw+^|8fXb2!=2_D zYl^^Nkgr1hWX4GL1|QJxQ(9*Mcm#$&98tyTq| zq2YX2VA9MTcH))+&0W(x-U316Vi}G<@~~us6?#GfEwTRQt{D|r#53j0~PKw^$3No+rZu0}!qhgqI&kt*;@p%)NATq_DqO8JP z&jK12IBn;4m0OFJ@4u;A11NTAPp{DN3oZ>de|e2V+Q)D0A-#GZ5}v&XeJkyiu-*Ly z+rY(^7|5F1zo|Mw(q5Ih{>}hXwwiN#`-8`(94oI$#V7lo|n*H9YKMc*ZUQaX^qy7&@ z`o?RFyRh)_sNEy3Zgzn_BAS3oK9%DKCm5+m0SscufV%S}@tSeTvz=(SGf|!(!L9_u zqQk%4XemfD1=>opTSS)f!I;*WGb@HJcuU~XO4vE!IT>y^EjE4d;o#Q|I8KoJ@aCqR zqBhFTezRcL_4`7{;Y)6Bxk5eDUmfFPs75XLUck>pe@~^&DrgkJvB|Kn0mA|)?2Fqx z;XMpIf&Au+-JJC?4th@rVgA=+3p3+DO`o9Jx`5qE3xH=~BCn)MDsV4m;!dQxw(~aeCEp@Zij`P z{JEu{vChQ%f>;U6Z11@n!%9<7gmFM(U7z&mBRwH^X|xD+hW?}g;#h*Qw6LnIvbRMQ z@oqY{l}=d|s<4x6Se~WgdJ<<1?9=ng9WD!Nw*>%VBT)POG-*6Z=yUA-xcAIjQFmF4q&IUFFTF%2^Eto~0(EPnVd9G>+GmWp z0~Z4&(#j{LD3QMIdn;=&6h$OV+K!-=gN^p3>8cQYyY0QC68#gX?Ds@YkdIb{~f)GCRs}NE+ezzA?K+gI1~Y|&)S;T zbsz6+3i~JxBhl8D(_5pLDBGNUao@$u0GgE1K;XQ)uPPB)PmWk8TQZ|BtS5eQUl8e=PKWTJrp^9k?<~j zyaG`1gKOb`hqa2_Zv7{uK}5%GO&_MlkIQs8is^px0=$~%a+H5F>>Q{KAr6g%-$H3H z&jQv$z(BPsL911|AI;BEOvgPknWW-Oz??rj?l~(Tu<>jCRFxKV2prm+X~-u_OrH-d z4B~obkl}h9yy)y6(M&T`uM)_sA8g(JS+p)tCwk+6G$lSd5f6<7t82$N(RKRG znX_=$#p9UoI0H0?TXMP6?jicApKNWi861g#a^UC zJ;p;$P6><`B(kgq)ZyopNB*LRJr0SfoBafKfOST~w-CkA3D48?;bcvIO&y8rJ8sj0 z!`2BKluC1l0NxjeMu84F=Qm^f?_cH}x>C!iYlP1O7|*7_-LLTqc&YwvfqP^0HQvrE z%{_3N>pWvM&!_8M_+Iy?!1k4^8vqHJld3>z`ILbDCWu(RaGnC>!lp00eizTZ@mzmmuzGkA{KwKWog{N@-4r6K-(UiE+kUG=8%2fPSSfh*ru zzSWLi(m3CDJNFOXr@B%d=L;cgsxc_hO^LYzQMg`Rd|;ou9z5#LZIve`_|?cFJmJqQ zNRlKS@<7!5PK=!??3I%hH^y+(P;}$RK;M)xIDc^-m_PoqXi+~!19T)JPIzzAB#xu` z0hX@?l@~&r`@=rWOwR@S+FlAL33zm?g`ZlS6mWMy^jXOv1n3DcsXpfgcyt7q%a_g{ zO}0XSYQEkH?o+yDJKw(txF^3664Ve4|9;$^*6|}6s75H9c#*(Sp{2Xl8u7=lm@*2I zjiA~QBJl-p#u82|DXOhBv*s;S^>0l2yWI(lY#KM1L2RGkqL2;iqP*YTi4ze3XN(zQ zC3+dIw39Ap@f_M*S)geE7&n(@@PO=eNC#J{e)k6|@uKiyovCGMQHp~LwQ^qOG%_}qMpT-08BBY(E@Hwavb zZ=0Gy>AbQ_!}TUm^e4DKD;wUvhk>kvMZOPIYgGm$^(`g&>UxEe@X~aN++!zE2F6}| zL%w3Gn-8B2Ez5Cjxxm+ACnUNtLKtwy;jv~Ty#0Q;RkI5Pa~+J+_Nqs9R6R@m2yDeh zV)C_)0#<4Nul3pBE)-Pzp7?0}Nm=SAp7N_Kt6dl?13TXPvctX@F^TAk1IqlNU@{bU zg#~v2gC95B0W{h$>=lz{-nf%*WnbpO1@+y@ahO9KP6TrM5E%65JB-Zv^Dg4l2c+ap zzIWC1lhdF}pU%Jn8DqkFqeD+77OT@j1d%|~+v$+PlFvRC@xAo>@-Mxtq8oK)AQMvv zm1lfgP-kW2ZjZ4wo#6`3NL@>i^AzGas@Xn(B>{UoFsADQFHkGxKPuI68BD$6j1w&L zgaH-@59hiwVL=Wm4meU-=v4kaq^-<3;0)wnmC%y6 zh-4}ExK{oDmdrBI;11ty8)WF)G|=>o#K#)J;;9F8Y7pzVOy6=O@<8_Li1^~4-V&in z<{{U1XGFxm>zu-(W30}mp?u01*&c0z15ndZY1Iyx9it;#%`CAxzoUGSe-bPs458sB zTHj{PQ5}%_M=tbEEL}F^2xct}JQCKzSZsGyp_&hwW1r_D%&ZmB$h=h?ZB6LCtplQm zTK5GdE@INog+)6Q%e;-u(9W(x$-L8qOrVRcBv=47uW!&-an$QxQ2~Ui{|g%d+yx_t ztg}c(JfzRr$Pj`((Ec&EHWio6*uqwSEju_4*%%Rw*)}^3vC2g}m<6=E>52PeBkM4+ z4#1r&h}Uc`$iyX`))bV@4>X)d^w(jQo?}`-`KyNsiBeKaHkI;8ea;Ty=Z*kyPUL35 zVcAg&x%jLZ^|kGbbXvR#Q9&{;*6)biY@+vib41lUL^>30JW^R! zT%8skp;YwIjgF5N!|vzK0t8#x&naXtGJ&?md-=0*xWZm8j*y_U5xf$>1t>6+1LCep zIllhNK9iJ*=$B@~)AN1coL64dka)qls(m^MW1VMt2RF3SY$F9vb1+WVDB^vvg?YiX zFZ^@3p($x`l5-gJo4BNgT3cEo$4Lhv)xA0?`h~OMWXeCUG6n&V7;5#`7zr9&?I;&? z*W@v>GpQLV`F%2s_pX<9AD4xexL`*ohY+zRck8t9eg^&91}}QZ$qFyu`Z4lOSB%!e z{=+9i|Bn&29BzepbO5g;fTOF2JF-K_2W0ob?nN1@YI}x6a8`9AaP%zfKuCuo)UYSV zr$MtP=NkQq021g21KA6rmo%b&8p9e5pe1Q4^U4n(1t0jYGd8f|jC`>WKZ?IAY;+61 z_io00oCs+z0LwA%;o5ustCQilRT%j@{I>HXTG{{>7ML~F6uPWx^7hHqH&GX)gcOTj4o zInr~7E-c9-HGO^*hL#GuN*q#^%H(1~Rj)H*gB!Fwdx^+ts<(0!btJ0B2G+4nI_vUm@4ID zDymGC>ci25*bOMPj3diUv;>iiGfrvh(0sBmlQ?~29pnIm!%x&K*q~_-qI^exD3mZ}LVA z4N}En?c-i)>0;E){`(2p(7I%qrEfhfg5AD_TTkZwCZUHG&PAApCV7iJc*kvW=$mrHSgvI-MFM=clr2Hwg! zAKRDrR#Ob9e(m27;c05j%}7IfJu5XjVPD8Z%_E!?TUngjzL*0YQ)y;yIa6d0c5on9 z!o+4+&E*HsmGJB2_HHKq#m7=E7I2pi-&1CrLs5>F>sd1PBd4ycWQ{ppxsHWNnxFxp z>ibr2BZ1>F3O-kv?o%u6VJ3ABXDSF*~hkaS+CpMd;A{|gBX|6 zeOr_OT)=SLXlNEd{MC_<44W4EI|K>_{l}2QVOtL74cn>+lV(_K6b zAsAzbmosZXIkHd%-I~fI_e(12L^=wpBRN6mR9PL zQ<)HOiUPMMgmCv%BM_cJ-*Ai_j}TT^;{2;pJaM{GVU>X!WsUbE3|FfOxcE9@<(z1>5LCV95h)<&w0Ru>#ykX z738X597xDmwjW;$PsXv_nrI#ry_|vn+?7!Axl81l6n=1i$W}p3f_S< z$hu;PZA+I9uNKJ&2?ALUc#(z!4jf6Mi=_LRg~wNp4$=vb{`vrDeBYKfZ!$4Yqn5t# zQI_dM_E_;!skc@i3q>P0r$9CIvZjA%c>OvdUdRv`Blg!DLB$0xtm`TB`<;UeGrm)f z*vu`j<&;VOpB!-{7+a4A*FLMXa-!LXUaUFI*e;vTyoonC)?S)D1$uL;y(-0FT`W{; zDI&D?K<+#z}nkqzTvGz*0Ehsg?Q+UDy*-f~v$y5@sSzEkH-9v4o4;(i)2` z5dwp<1I(wzB+?M0@>|}+mK7Zi)vD4L3v`nK7bh%6UKMyK9%lJKBV8K=EHVl#EGN7g zu^@1ewh64t6>BAB`2m7|_|i0}!U*zdS*rTv05>j9h*`T{&#`96#}pZsbEA2qbDu}@ z|Cq23G?2Mm6A@L;`)vs|cs080B+49hTZv8O_y>KaSOvIew(`-bqGX5>+~!>fO#@X? zC~qV_D*=#}zl6t&Ud_XQnNC`*ZXl<0|Li$I%;@`tk9B3a`fz3yYaPn;7nUmPB z*5r}#2jlIStkZNM!K>NG%u5Ec){30f8CFt}HOfQ&%3oc;@q3W2lxcj|cp6U_5)K0P zN5>ag-#%N1c!sF;9|tM$2v{18Sb2ghV=AcfY~w0T^Ta!;pqIS&cIg;VhX^9sV`M2? zrM9gQeA~9|U-V3$S~+d;UHZlsAI5|Ragz!k#ku}4H^QFMa!G)&n62_0vV7hYB(@I8 zCtZ`7Uxto`8mBAt%olT#E#z4e8kL(`M}YTt5~e|sq5r!m78oC$;c8#(xIZhxyxBKs zW5_s^IH8PaV^|6i_@{x^N!wS>xpNnc^7trQ>nw~NiD!Z@JWhGMTUNbRAxMk1+y&4* zt|mH9?e=}a!J-|k_WRUP@OTfwqG4f-fuz})M)It$5)FwX}3+1HOB z(+WQ@9%L3~nN9OV7c5_+0!eDXEYX&LQg<`{+w2l!i@Vor3H!s;n|hBG!~77)UI*+IP7RkUqT8f)3CgD>9j42 zm(A`ctURFVmqN^&ziz0-toj}bXejS~hhj)OkMYi;d>;yylJP{a2fvjT{=0S6E3lu9 z2}Z)neIA{73RY{oS)SKNhfntLT=U_%YQSF97;&Gmn9)LrA-!Xh?EtIQscD5X{?0Ej z8}0y;*%c<#8HW)+vjEFnS!KZ4wRPcQPyS7~W%iLri3GnO_j@6J(_nM8UF5=FBLoiH zxmlSBCjg{OQFmTlM!#qc2Lco+Nl7`lgMfEb-E{ww4l+l#oH!gcb&;LC&)6ZJy$9^e zslT`r)nJno6t%|-J>H9~-`lZ(K0z8*GBhSTlo9JJ5veKc4`H1HrHPv{bQII?uA>Z$ z^4{GKeMJ$YR=e>9k(~dDYsuU?R@UU;H-C;XAzdWOGc@FCOu@|?z(4QPTw{IRV?ur< z8TP~}wQgL>ifF178RNr+fo6dAw{|bc)y`(&uMUb_*x4D9!{AW&cpK@hmWrn&_Bsexs8Ux_@%IGe`eR}DWPEzoyi+lD@W&!rmlfW-J5 z6Eb}rPpVqvpmeC?Gf96+I_J#hoDtL;LSaHl3ELMaQfCjd$h{e-iSs)4!Qb?%c~^ml z!zgNtPVt;Z4G?otXkk!^rquZh?56sE@UQiw=-u7 zMl@HV2_DlNQ*8O@62)cA7IJVp#Gy(2zTjFI3$}$hG{iZY2k-VBgzf_K@iDrtK#Yel zId?8i34(OzNrH^lkH0>B@WF*WLNKr_G4sT8B_}g2E~J9*j4RQ4ZJ&P}QRNbwAB;r$$l@`!D$Wqh($%&eeq05j^V4+p{L8 z69mv$O~epJ-SQc*DhpgfRlwCN0?-kct4S}yP#2=-`z8)Z@Y3`-!(i1nZ7$sBZr+Kk zM!{l`JV(zgSWACC?E-hbf_rD*qkV|H06HcbF?ulghsgIkg`{VQGJ&Y?C8yjzl#Zv} zd}Jz=C=8B z9S=6{AFPQ5PIl;C-Y{fd+_L=<3oh>0HMyN%5Z#gTpIf>wC77gN|DP5hm^ssNXpVy( z`x+0tB$Yi5c1o`S$b?A3A&`+>H;;dHYP~PR0WXqd*uu((0{<)roM{b0RQh$3ORa09 zcb02AW^^qB1KGo%Y|Hh7;&AiYwLeSXl z69k-3R@pj9MXUUM>%U|I?uAF)Al}pLlLetd0Keacx_TR+E^vYDpz_BKk0hRT@3v6t zy;`TF75kCMn2M`30<{>j@FhZI^kYVxc4#9h1zr;B%YO`b*sAZ#+2>q(#l~cVjm(r| zg^Wse+CuCAfb*wEtn#?ERX9qUg1_M<|8u;zM%Ov`C?hi@Qe1OVgfP)rg?iCZ%5Ghd ziett&%kOmz#boR>*gVx|ee_Uipdo!>f2HzmB2!exFt93TVeu7LU&^#sho?()Vu(TYj_q?BDJX=$)WBiD z(l!A&oR=MyYBlx|``V$-5T`|1%V@tLdiy>wsyV zhYW+cU{o1ZbE_%(m7pXg=>=NlNTm7yqv;#iD_gp5cWm40*tTukwr!`AbZlE4TODJ^ zwrzLV;mvu!=l+5Qsv`xEdUFM0Gr->knl@3e0F!IjZ#-RDcsh{Te4xzCfsf*vNAWNV1lS8f&>sU?vcdeb11osnL)CD4}AK_H~ z+W>Ib9)c=u5Bel!4J={h4`W?DOnR zZX14j1d1?l`?MYI|GccaX_}|BuVwb1KfhQSpOZXf!b(R-Dn=DO`f*XHfA&zC&{MIa zm(@$PTr&mS5@=K_?--}|t{VkM(Z`j#5GibFYtv);tcV0>>Mf^VXM z4JB%69(QES1sT2jkkACEOtBAFlTJ|qY?xT|44_+I&IMnA{0AqkPKUDTDGLABI}mto zGUGU%IbW)`_q~S*kCxqnU(KDM|EsmbW@$s|F4@BH>BIHf!Mz~GoIf8i9)=}fzN(pxIk~lg1i5JAI0#09-xnwGtZFSlOo*#g;)R5p+rP*nKbmRA3G) zyp(VS3~Zf^l!aQv`;7MDXcAVGql1~?T#0q^Mt9|gt-|Y4&4Z2-e#dU1iKoo7v&kuz z%ajbg{m~jPrsZfSFt#+0oXve;5v!QXdw}xxr;T0PXmYk7KyN5?S)Jiqd-56V>sg0i z^(R*A$*(eU+w>58){lY!p9%b`$_pf0re%G%Y+l986FrN30gqv#p&1Ek4$CcnH9e91 zU1wObcU8muS;!#Qy#cG|Y4N?fiLA1YW3dgC#2qN7fj;+2@s{pBLUdEuL?8h?T|?mj zg;xY_AGVMk?RZtuMC>kC$_h-A4NxF>QwL1?R2jD*ao?<>(;4!Bsb=C1$0+;QA(ZV2 z>=HW$!~=xtH=*|qxC?OBWT$sEw^x&5IYF(PDF2L*%~W9!;1b;(vfOVs9fqShDanmx z15LvmuuSTab@R|tg6E0&BI30JD&+Wc9I5L|UGQ(`9)>O{r{J~i>}1c(M#z>Cvm}@h z*$B|hZ9M}URbo1RQw|1M`9+_N)AaHF1?6yJ+$Yz!){@7j!YTQiYcuXwR7O!Ktr`Iv zFNvG@=bNs5n8#(!-g;N*MPOF99OG+3vp-Y9IW?$bGVVU7q0@j(N-4?b zzJskFH&dij|ZVPeh17q&Y+9@9>i-uL6IN2d8|No%6>^F+B z_@de5Sz%mxT@FX7&GnFTh~!B5o(|J}&B9|+W__YrNx(YmgAiSkyu|{ReGe3oyn@Ie zXM6(c+4*;h4QvfJMeRO1>d`F1Sjheo35oS`+ndmVJQE+(DMfx1F>)Ph%K5@>Ft08s zW$eNJRDi-~;l5@TWP1<6iR~W5IO3HQiqdx{F>nP`)TFEiLz6#^$2d}kE08U%)J49p z`Wifv5sfsAOXfMD1bu-%&XM)O4TG;4{VhxyE05Z}-3fAz&5=cr#W%z>0N0lyu!S-Y z=N@j+VPvoxkpGRuz!YL=H&@Rj|j1 z?kWPp8u1{NOu`C%I*wG+&{S#0_v;XW0(mk=#`uqfpez!3E)M#SCRr$>8-9tjG)}%S z32D4ZCad=V1@L{J+n1<4@=%%k%5hb(l|tH459Yg~9{U-d2PG!?@)Y-jP+ls)Ea`>P z5EWD%TqJ(*IiA>2-&?@7Hxaf$wfpGLd@s$49>m$J66ZWVPn#HjdXUngZDFhqGtIL@ zRHmXFvs8K@*fJ}9C#&H|$VU`BcaHuJ&AG8eNAkg(k>EXH7FYYFV(Bhj@Mn@~(JGe{0)_pJaq|Edf}C`gwv(}FGZaRId9Pg0ESetG^Vi&ZEv zuy~oNQ48Q^>gTS=W1EF;>*xHN!QIz8TAZeU)1CHez-Kn`-(4ela^wsZmUbebHnKnA zpU)hsJ$v{n$QJr;m@p_}kFT`K1%iTqwn%0t)%?ugJlXaCZg1b+O!PKd?UPx=niEMw zu7td9-zSFewiS5UjEwM@s>}Tk^!+)GvtkYJ63$fyMZ!yR80+wNYEFYdVi#%8Qzl0N zR1^r?MDd0`Ka|xYA&p9&x1)J1{93s33|b+lnsoUHB|45!^JD0gt8mY=5$`%hU~=}N zf4DbJwa7|r$Af;WJERMJxnaEIB6Xd6j}Ri#9Vrt1A$ab(&@FiJnYUuX65YGAq@>ug zeb0?=v$rtP2OI1GVzUI!{3-yEp?ZB7yCMiyu&KD6rYpxg(9zdVO-;3SXgiRewo-4y zzJLbPsg_M4bYr>RB9TpAVYtS)1Y(DVs`o-#Y+K&Z`R4D=g!JPe;Q@c6+ndd3m+0oZ z3`qR`0wXT$_2(_(5FLt(aYtEj`KeoIb?e;>$f_VphTxRi;=WFf-%wi5+hqG_w}u}` z(CKOe`piEc-nMbXP5$+IXLff}6#MhFSQ#J@6sUA67&-n^NB?6NtmkZ-$ouLjoslW^ z{pw(5kiV@5v2te-X~>fD8s<;pJiJ(gS{>F7uM>Kmkfe$I|dmkbfK6Wcdcx z*)z?$@qT;yD*|u3&IZvm6NhB!{t#;0(Dsbu%xqu)@K|srZ~(8|kKXkz7jo)HxOD_O;WXAt+wXh$LjUS_#LAcjjlW`jj4b!E`(a64;~SL7F9&F7}73z?h zgW6A+i7&QEy}m)d^gQFgY8 zQ`;#qIWGy{l@^|*R4Ak0SqXochb@)&f^k4NaoF?GQEw}6eu0j@wq{iQ#~#_~@xTAB z(dpo@t98p{e&)}dsC>#?-;F!7mo(!3Uh>zP1)YwuNID+pn==QU8-awUh%GS4f2f|90%t_C$2q^msKA?NZ=+8t=H(05-LJ#pQO&u-fVJJ)aBN z7X~nd?UGUvCJZKp&=2UKuzBbTUZ2)fw$b2p23lXRcNiN05P|VbXQl*NbD%+2 z;B`M!t6jjKD|Wu8L-lqr&%m97p;ab>3gmhiVBPjMyiuz*D z@OeZ$8TVI8ufB@vfG^IUu(9TIZ!+Ov1b_&`Rp;#;fg8oAt853X6h;1je$7p?1xJIkh`DPOX3{vx1rHi#i&oG(Z(pFoFCt9=Ih4mkaVLNna&^_qBry z)Cekw_toR_-UE>K+@9<92duBP=h~R3M?F~dbQ_(Au5URPu>QHKu&cE3u>|478$=Jg z$=!R4FmU@sO46fJip^8FkSF zzdQuXR9sffGWP)Fls8lCvDqzK|t+G@ShTnu7-2SidQ9jExm5*sGmUW z7GK^Wd~kI{v6b%%LMEo5WDYwt4u!^QSu+Yt2}ch19P@d8)|i1Op_R|*aNMK)3&XKF z!7cO0(euvjhaj6pM4Wt&YlPzV9k$gydW9ca{Shf3Hr_emhX`Z>LSsH9PMFYIL2@E3 zg1ZT{IFaeVrD-22tdW!t_U)`0rK#Krls&2n`D}lEHA2)(>Wv9~Zn^uU@IORNW`EHj zVM0UiBEFe>(eyyCt>~G5+rML~#-n;e*}p{q`~(Co>N7PS;JmS1d^JRB4xy`FkzkJP z`sy7JZR$^w5peMA#?(I??`D&nvA0G!*4iqtrqHQxYCfm6J^#(0UAWL25}C!qj0I+BSPs2g?%n8u}RLV%)+0YPf+B~;?8esaJPe2L*GJ&7uU z2dnaWgDFX1MbHj6nd$TNyJOhsz4&tp`D1pP}{{R93>^27_=Z+&32ymwJj=Fy@hd|Frk;wLOCQ+iBP*>LPjBfRjkwbA9T z5#zO&@bk0+pbtetnR8q)r=eNryFSq{f)XfB{;$C}W{-&}fX+4s$|?~r%zbz2?2 zKA&&?_~~CjNDF+X=xb?rVD_1&g@_8nibRj?4+`XgWQ7OgPeQl@33-}t*R|V<{W)`^ zxd|5<@K1u2Q?Jrrt!T2a4Y)u1+0*p``_!a%Sisxm^q%(%9`m*EV^!yZ^Pc7NwYU2D znK>}W${`UGs%gC|{@eJ42xocfk#zOQjUWIKeaz&pGo_I03-Jth5^Z$8ka8uXZA1KR z0Bd34PD9btyg>+)1SZ}{C4s8=#yQg08hd)+dB*XHs-ghV)v~6s`4c?hmyrTj)7+0( zyfxnE)@T^1f`T(Oj^z)KJWh@vc(aKIf*8uf`xqSnNSRH^%5ljf3dtlFrh4TMv9mOk`ejD4 z-^qE{#QD;QwXk`8J1st};WFY)kd7*D_7NFUMGg0hWJ<3q-$$(|RM$S?BLiwKhlNaqRDhPy*wD^({Ga=PNU= zlBj_x=kd&O;4-CzDk=BlvTkO>UO9K{p|Qul~jiTR()edvmRyLRe9c^vy(mwKeKTH5~ zx)bJ~lWIjtP7a75`uJ7Py_0nKPU%3owE`D!$j%D%D=ltkhV=39?lX{DP6dC>pRJs_ zwE|^P|AtiS!p@0=4+CUZ{8lSi&}8~1BjC+Wtw8;B*eML7ng4dh#R$URp@!zJLNtLM zHkv{Bmsse#_Ja11{mV%M{3#YJc$JZLH6QpOL~ zl~Nl9{fTd=Tfkficeha%4Odqn00A*cWbhdWw)63Pk&EKDZ@mI&XSv84 zhaX9Te~m0ul>31YGM>>ueqaThO2UIIy^a5{Hz6ddOH58jZ`NZg_S#Nzdyk-0h z-rYbx1(p*(WeH@B7)Cg;qcIN*K`+A%8z=@%od}q&U2-;WMq`x;_c_Eqjs29gX%5ym z8>v^?<0LWR{3!V2506R>udjxm5}v`~U1p6Rp$f&H#KH-$nUM&-E99%-L>;*!ZBHA* zz7vgJueXUf9uz>(2g2v|Mj*&6Z4(LVdBPfq_mG{{)eUx&+i=BH5!Yd7POtPcZl7cxiIN5Aw%2JXWXd(xV_{x4D2!dogb+c8VkbqsHXXXZjR_htU zZ|Gt=G&RcB$%AA`XEr=V>Thr^Z?ljnMef8u$*e{(#vD5 z?+_Z`hmt$t@YnA$?~=b&80{U_Qevv?H%uKaq%HkV7eMhj2H7Sz7hck1noF}8*@ZkN zGbMSHxsGxDpMsbGQ$+~zYM6EtJyhsLFCU)0m)e1@i_$MYZ~Tz6j(c?-`}3mHrcN^` z4cGY*AJ^XNITc^csmHyyT$F~v>ELqL{x$RpAR^<@o~IS|g4|M};yp5#4nQZ*UxP3L5_~?*z%o)!ELK?EP^aZ(;JLdGF#jPhsDG^(7HfV~U9R#{ME-Y!vG1tLp zAc*5ih&?ND!vHJV!QW!5*GxX_y*wz)xWkI*GFbq1ZzbQ@Kxw|ljH_!(J7QQ%nOb0G z-}uR~4bqp8HzrISlrdsQz=S5BZs?-CJh=M70ws1xHi&lcJ^DeLK)17|0kW6k&){MK zgl;(j!ASgmM*rOytwnV~v8FcfZ+!h~R({XTJF z*YG@k0e!C(-VBvj$aVhjdTASU!m82tiw4|j;O!6nhyd@(Ynw-*g& z4l;eF+G$6lC&BTEivQ`Q$kTlvK$$fM!~rP4>g}7U!91~_57*E{F~NC0Yi1M{@A@0u z&*@v=>%oJ^>U)37NmN(HAC)aG7F)CV=OhMdii1&%^%>OYi*Q-CqX_RFNwm% zKIu6#*vn+Q(c#dUKltZeL-C9iD%q$n4b?g$z%}+L@gQdxi@<}YQ#-M1N-0?fGup07 z;7OH6HFRO@h9Dto@tYwy3P~5*4vt@B>n|MDdb%IE! zna=!KQa&j~BHkX0Zhc}ZOAq3J z%-BYZojY%IO>&z(bu=21#Lp7;=izAG9&S8-kv!)rU;#UJftYCdi)aJUUhDpqUX8K{ z%rNlI;XK$2bbfm&`1!x3z=VEF7G-=RkW&uU2?+ZP4WIoD!EZUiipTSztp9#=uvD9m zgwtV$@4Z7B$?7LF&)5i$zQ8FVK1qK6=;GbQrzZGXGBbCpP=(4X2X2z)bN^} zX8hy70$S~Id8UI2CX=3Uz4dn~!mS&4j?0TDzHOBpUwT_0fyWmo?1By$eFH)X{Fg%; zPJe%ZRR1U!cYy?ee0KQ?l=kH^8-aQknE_S}I{ev&ihQWu%*zjgF==eau={~8&MN}n zBbLqGQ(Msa09`7WAEJkZ0*xeYOtb*T2ADB;`(1Y|38!6XVmu~}N%D>PBEI~43Rx#@ z(2&x8B0!xY;IG~sHWJF6?T>?*!p(J+L<0ls|4E=B>Lz( zbKVJruV&@W`UN=Ee8-PZm5rAF2{xN^WXppF{Lg5Nu0OsKEsk)u*a!s7e&SZ8j+OwC zt+~T4m(DIsU}oSQmvx{0BT^U7?AT5X_o|}6$2PL}Z69>1sY{tuo;QntTC17tbxV9$ z^1JU@W-%4(erkF?6iFkfpKA7S3w{`sY?_dJ3%0B>Tdg-U=UETcgQut<3qutzkRw?j zyr1#08AP&6mOIn;5E3`0JXSB^cSOWj)=fs2!MKLP{X<`@3w9ATQ zRp-+a*ZsfV8DJW=DQE+8@wi+GQ(nRAw9e$qSV0E}uInEMwhqOp?{_guL8@?UyZFQ< zwqE6))PHpl=Z*{j{iaW>zzg8qlTwV8607TCTfi}E4MAzaTtQiJb!=qk6PgUf1p~fR zL)~n>tiliG6ub^+)ri^uP;Jhyn{B%}nL* z58Y*LxK!f#ADGF2a%AkF@E`@?8XTAA$_`KG+5#FGqq3t(W)=} z7bsgZwkC7y{3^M}->)XmV`mq&qwO#LCH~|vvjNhgf}jm`ZM_6`a3y4L^y}`Tq40|k zpV(*#7IlK%O3w^UM(MKjA!Uy4+8`MxVR2!h4p?ZaDp<`SdXR_4EVIUh%rl>D!Lq8? z9My7PicHn5CH|S`d+E+U=Zmg+s#~rBk9;SK)Ks}8KCmfqlb45+2kIL#8&_vi6Snx` zPFq;^y(-&sv=S$C?*?c)y);soU-nmuV(H_Sk2##U3pT3;#fXnutVJ>@nmIClhBAN7 z1?9z^Vyx6{d*-x?+g(x%+E|@~O`6lIR!n$N_RdZ8pwnp^S-8Ofd|r=&=;F>4+qk(* zN;h<|3gkGmNz0s8imNK)Nz7%XDWJms0@ybG@RVY30IK{QK511$R8NZIrS^3pb@*lq zoP8ttQbh*SO2Gkfh#sx!morM528{JrsAEfld8b$vkc#9A4{J+E$ zNi?@=@92fXWHZ9(%3wX+c>IQz&shZ_ZHgT9gw8c5>ffDLCaj(fm&UD7(9Q{@QefGL zAho}=3{N#Luxw2*zk%Y{%#}+{NjDLB<>Cm=mgaiiw)SQR-K0JZU=lw|-JjA%gN@G; zg=T$DBg#DkgQ5ZzW=1STH+t8siGY?`+$w}i5c-&etBfs5d($5R}`5En#CkyIpp79@j` z?%DJ;77{oJ@$s#?wYZZvEXJYdCt4o}+4h>^THSz*fg8gbnHeFfUWg6Pnigdg!y@;W zi7U=EI1b6Ms$b41&gRZ85pWhWWsV_H7n<4Co(8eIoe4wP=pZr0bc+xYv5iHibTr(| zW$L_H=dUQ@I*&X>pM8AS%4$r|ymU z)*{IAAS4q3+4YrpG0Aw@@5)i(4a#q^tvVA{JI~}?xt>-ijwrR)7VAX^H2dp&mR8ys*VtHq_jgvuK>R=IH2xsgxEGUVN2HYYJz3FJHwrGrs#>#ypzi?c5+6grbuc{FB|GJ1)&xt;54 z*pxz78!gl0moX^^Q;4}2R}Nf91oe`{=dDAr9p~yjsOEjEL_|new$dJ}a{{Gcn(~>- zE!vW1tP1K$J^KFkPan1@)ifd!Vq!Kpr610`OQVxky`6^L3B=tpw8uAw^n7dno)(!= zHj=^GhSo|OC6P|ZgcVX$ z%sw2qq`-drOE+umrG#3IUm)G8@Jgwhs`t+gk=0`BNY=PJA=LIvEov3B9H(~Di>KXA zHDk1_xZz8b>u+H4GpTM(pEp>%PI&ahx>~nR`Q=+#Hq-wVjjw&QDY}`UaB4MOWYk3Q zzw#QC)Y#765id@un1W*U;RuAPtoZ52joAJBAHst?@*Zkd(lB|x-reHqv~A?A7{70- zoUS!idUbV0p}3C3zjeW)^-i3UUwX}dA&mG!X&a4Sq}g|*s0FEui-F%t)1P!6`d7Ot zanm~SK1=rZ=#Oq09fBwgNtLUW%11wn#;VphM8A=^+zyl%_A#v^Jv2MzIOR>i>W4JW`)#|>xS5n z_s$*4=y|kr)K5*m>o`6(zrP$zJt|*hj>|AhU(R}IWPxfs=bEF{j0@;>*NCM85iAa+ z8>CAXSL~H((AY$dEd0f7ZJ!46RZH*EB_)$pZ`84_TD9XXR^!OD?)f;GnIpEX>Vm~p zo3!$_>M_m2cKplb- zmEbchCsU3VFT508g;kCIL6m*$V}@CZc2Im#=Pp?wZuQSES-Eyjxzt3Ys41ox{+x$e9ULj1d4dZ8BkKkYG_0n88)86xBY7aB(K)VHW*2)NU^-M{=+(Fyp@Smf-emJ`@5*tF-w?ErVI7aZ9><>U8kG=Kx!L#S3g@gx zG`G_fG-)gF@mhA!$v6JVtcrG%wMm$@5U##nD2Xq-%^>Tm`c+e^@ynfkq~5gNc}F3V zoSuwEkl3uZCsQLUqv$SLz7%TYMpDIoIEz`g7ZTQq$5!9OU$pq8Pm$WnucGo20jE1K zp1|6WP-+<{WRa5GxVZ;gpg$}9jt7Z#5CBTW#Fa4MqSBw#wC2R;Bj>q6X*KgI0o9LK zX#O%k%pP6T-tc2Bml*ZX(zmBZ+KWu0;woB;jb-8<6et6B#8Oam3zT=v4uoxR=VQ)= zWF)JaWqA$O%A)lT+i1G(K+FiW9OGSn8xcVjL5|>ju1#ki`7W;GDQapyQ8FN>cu3$? zTk{lut9@F@T30IPa4TTKouUuZHezv0Z$sTR>PlQf*%OiKZA~@z8j_B0Iu4mvGFj-h zgT{;i$mK`f6GxP-<$+Cmao!9v* zZ8%QHv4(S_T;UJ|x_Id8L(2==#jf*m64J(8Yx|8GGgpI_cJTc~@w~(}@tLQ4 zlfIqJ0xDC>^*o=$&f|{z$+!(`L)7PBMQ^R=MI3X_N;Ql|Y~2vvZi`Y_Yoze=yxQsq zq>_tLqGyiTMR|i_B}UP~$`bCxE}V0jP1nUX8!;05ZF9~*b+OI8s*@3wUUnXmH{U7C z$2K$WV4JY=C_A<77k-<7q#nk(0P*+Ydwj_lV7k@xD_dZ~xu9-mI=Ue$wZtoZ9=0GJ z;(ygGepDRV-uU?;e5zsUOgn194Or)_T)~bjd&6%52lhEjry4VUn<`s^;kP1ZTKAkv z0Z0u&hOEMhI^3Ct59(4_`?LD6`o+)RSTC)JtI|N#^rNOFe*mIr;XeGylz)YbvF4!#Iw5?s0+xX8$_7AyA5U^ECPpv`ql6+_c9lZ2JlZ?JQp(t2Q5UI6QV#DsY8upcZ zbK?12nGG$Fk29#{!wyrE!|H9ZwW6r_rD&;eHAm)c>As+2($t7F@5@cSzZi z({*dipmB2tYm9wk2v|9|48H*_8Ww!a4p>s)Ld9|i{mJ916gyjlUbv0n%tzkCVD|c% z#SWMq>F!g-K49VH5r;7qmHCM#DdWT2Li-w_xvH$)!m8OCiihv$QKt7CUU<_qEv3e* zTRIp=#ycpuwP*Iz_BBV!7bigLJM`ps%mvismTW}Dn|^q)e_(YFjZlG>KxoAFupT`p zSZN*>i%+tlA&B#cF+4lq2fRI|6_>?AYkIO?F*9(_Y_Ev_t5nhhOjC@*3@N~6Q1IoG9$Q&CKyFV{5}dd4|d()bweR9Cre#dL?V z`mngK`{QSQ{|O-|%`<^LLLf65DD+ke~VYmn9BO;O@D2H-!^B@0?&XdDV8mSRzh# z(JcBhZ(z+riBF)!xuEP2Mt9}!!chv{Pit!GV!Nga>T7KJ#Ar<6j4RIY(bY28LQHNO$?b3MaJ1Nr!Xcmv}YwOtJ$>+~Rsp>_f>h+UPBO^FiFs-gksZZHG zaw%)DNWYv`?G2mqz$E`G{&>0TnOA~iDHSGcJjwS~mH`b+d(#yz;8yb2{NR4~3nz9=nxR8JY*j*l}xx;>sGN{uYt( z>rs`t-Si0N^5IJYKWeY>#MmuT3dM8hoDqiBYU2=?rsHiCTa3S3HuQY%wBZ4G=8q| zP#;A18o=Os#93HsI6565DqOjtu_CFzox?=>u@q6V!<9wV%@tM+nBA|$D5~0^YzXDR_)#%vUVW$(7{WEi+J%W_O}tMe8s;~ZR4~NwHvqUJvV%xv4aEaQ%uAY z#FqR#Ul|MCVlCCh4xct1OCq1SzPif&rY6bauR6(8kum^w*~>LSaykmW{z~F-AsLy5 zQOdZ>L*pzBS&)v;H#yIFrM)sq@wXe5=D0vOl#mpY=Ti_JM9+39;ab8fbkiqFJ}M|< zL_rc{r+LN*G@7h$I!1|*?vYdyRq0{>cvs_fZOL7?u^fMrAgzq9rpSq1$6aG@ZBSOo z&8-mcura>Zn(!>WieImL4_ai)%bfl--QBwNS68Y8dI^p|b@D(_kpx~aoGidU>IfFTLIE;FBXGgHSyxcOM3I5sQ_ylE+aIy=JRq$729)wZbdM`hKZ$&uO6u`kzS8l8EnW@pRT*+%C0s@KZ;WVxU$wLQ>TncEO%QNz1!5kYVueR5!!E&&RH}0Jyy3bV6+Ut7fDPyG^}x(@gLt~Hny+RI ztx>}o1#DJJI--n(sA)}vCU#7U{B;Uap>T$Xi$S|FMZFWtCsjK1lAxP!pD-v@`FHSy?{%4JnA`2n zc~hfzG)#;y9i*xF_*+?ZQF8I?co7;dh{WSnezbCy8H|K8s(tx?zD>xKf;8r~Wk%2- zQrxUU8V=p9+8T{{j%TN=Rjk14kqZKh7xGfuF%Hw8^O+>1zG1$zS47WPMUDu<>OTKd zfEn(r*EI*mC1*e^cI8NsRA@56>L<;U<3`e2nAU<{T=oG75(Q-Rg~_mC zYS;BDf#gP)VTfQP2>Hww-MFu|KLph$#VIi%9h_u6bZ+)rWEM=mZc^PYQMOCpn1*`b zb^7%e`*>uWX7mO)2T4y0($d|*-FW`T;x=(%Hp-q2^T)@>Pp`aZb3mO{8>5KoHQeBB z%?6%-COqQ6pS>|gLC(v)+yR?_lc*X~Aw?u05I7r}7d3N78}nRmm+d8J*_KPxWxeb6 z7BfSoo*~tNQz4Ro4aKbLQb{lC!&GbGJ8#nZ@ZFfO$uY7Bx$!;q8id(Gk`641gb;ht zQfQbYP*t&dWko75V*V1eZ6n)=&IbgD>B{-B+iA6JDCbCqqy*>RTb0wMqz zXMzV>?iR-TM)MORb2;WAkiSRj4FnqN;%pmKD0J?XF?vV0Q#27#IuXvKR4vvT%^d+s zHf=-$%^488yx+!zUwN4Qsh2;}D5e(5H^)r1Zl;6Aj*^ncAXefh$GJ#QDd%3s=;b_D zF0NbcDWyksp3)L)S1IZ3{Sr6EHh}qub^+;OdF+UYi0CovX{}IDaA@|+D%z{q*T4BW z4F=ztKbp%rIFBS@SxPRp%V_C0*SX0smN9T%I4~?13Pl}oIqx6{Ziz3z!qN!e2k~u; zO$D`xdOf;1iD{r7u+Qh7Ib}2kpaPt&W`#;^;J{YS=9n&hE$9N?bJ##sEphGzqZ!1| z1mOqK*nAyPc!%E^ko5e79j{SpeJirbCu>4kqDh!w546_^yp*hgW$cBl>#_Df2w_Fb zRL?;DUO$&b8jj-Dhz8&-Ri?D%%AC|T+*O7}#17?X`HPBRA1HlUMv0#`tgslm%jhP_Y$Y9a#>a{n z>?*(sH`ZllUUwH5VZ~I6%7&;Sm+dTi0sJ?|NV6Oo^N*Ba9*Zaw`Dcl?H^LX3?|&2a z;rY22v9D_V>SqG(&E?I_%%)}&ovZikN^Un{s()7K=lXntm4x4c}lQfQ); zp~A{(frIcP~am;R6bM$I{x6HT4HD2P6ex+ z-(Py&1VfWzpA3^lbKH_Xw#@f` zas_mCYz07{4NQJ%P@x~UZp0Im&AC0l3^y8LP!#mvDWhBBFP!EvL$V38`NtScJ>=rF zWDHomT4X;N?pU++M-DBHD%pnMsM8eH)~bnEcsG4m;MpGpghWr$xXB6Re__edQJlbA z^kDRvi#?tsqRrWjM*l@HJ}w62B$;6NPrhpa3}qYQ&Wy9`XYD{m7XOVT!_&PjL5s!G zP{fd4JGC$~+Z##VRI@9qQgUXz6>KnB(ya4%+xLo};>Ky>T*=8bDyea!*wf(cTdAr$ z%T}@y%EoNdo;byj?>7<;91*7H9+^&dRjd5g1G0R$`f6+I&c3h$6y%CQNqu-5>GXn< zu2qVD=$^;G&oPn3M8pd)^Ei{9F6<+SE;pBurr5ZN0JN?irmV`Ov)SKtM3c?!ntq@) zeW=Zutg_KD_mkQ5%4f73>ke_!9E@S7BV;1pBz%5;a*I9uK)GVm*PVRtZfIy29vmFB z4_vn@SJmBokp&#{a&*(DPw@L5ji-{uT`A=MR9Hir1d{&26jp(tq^Y`I;L$Bbi@VA= z<6l52^;{`pch4txPrDVT-!7PuDS4SZ`0h8$4zm7L^e~f-i@n*Otg5v^erAkFzIk5? z{a1KgA>?(8VyBDp&G4=UTd9Niwd*nDQxqLVsQiAgw-A8rTZogNsnn>d^E4rIC;b0d z0J{Z(1vG$ySNhtY+A;<5J*icSG z!_cITy90WTRxbFt}vnOh$U3xgY|vO`doEM=1`k4O#2_|ggp3M6zp!8#e&GfL#9>q!QV($W>~U+N}) z#JCwB7n%Apx>(OppPT&>)0srht>}_j%*;OEzN6!spR#6!PW55GtB>GML(Gj`SuHff zTZ&ZqWkO2AD#oFVfb%F>WZXO)qk_h1MZ2-+lC}HWSbNdq*Q)pi(A*^g0fnKB&zA>^ z^}f3nALTM-;^JypV^ziRyghBLr{XHn3umfIQZ-GAMqo9DW~{L{SU;hG;PD>ysx2ya zbn$x_?K0sOe&MVc2OUd7nvMni&;NOgb3?JkA&Z9jJd!dKNrBIk(+L z?^k3>qq1avWCk7RYW3hsSqtrz>M$>ZrG)-?S!XUyP%nrZV~`-|YEg?|Wx4X(lD&nP zUF*VxG?BgV^?{4*jvM0j*_rqE{~%(T@vn~I>gfKT0U%p{-Aw|Y0B777HZ6-#wq%&D zvN&$=G?iMU0N-&ch_#uSx`oE|yc6eN#)iCeoWE^CUh_0^*maf9HMG$GW9%&hs`{R= z(R~Q%ICQ6=fFLE(c>pB^>F$z}MpE+7A}Smtq@_i=Q$RU@2+}DMlG4(5@%z8;yk1fD7ED*?Ao5A(ylkch>h2T_>(MDU(2}8^?im#|4+AdYBhzl0^D~P!5l*)h> z_xaxrMrfyFu#Z9@RUboC$Tg2Eaf8N^9rpGst$H&57%M`!+@AFC`1sqAE`i0WV}Tb@ zFJH^wE_LE=*i#!y#N~ZL;dROlS2^i1M@bT5qBkxSv z7m6*16uNv7t2T_MUtCFaxMy@r`T7@L%j)XM%jZ-bIU=J^jz)XLulk@B#fE!G_eNWz zKWy-dLs|v~G$uIp{CntBeZjD-n%LUh0!#c zP0+^qd*PZfTd`kra^ff(&D}7&5+05j8Ti}zxY@IZyWh!?ByFc`q0*L==Cwo#xdVOy zKH;Z6Y+&ECtF0}LjA*4YmRMMh7OJ~pd@5B)32}EAV63)NZ<+3W*>KEE%P3-zKl~)= zRavTz-8U_%N0&5;Br2^X%w=z4zLknNlf>x6=%XDLOF1b@F8fc@HNPo`X1$Ca{rm0Q z=fdi6@x6YidpXq?6qn&qb5(&j2nRR6ETew#?>@HNr)F}^2&_0N-fsd2r+h9RHB1&% zT3gTg|&|*%*sf zh7~l!2dWs=J&!*KqV=(!fK66Fgp&r9^t;hP1^BL5UmXGg(bQkM6XnD_=CYDoKx%;ED zQ|{E2LHjO+^f&bNpJ`XV)c4aLf(DjiOf0)`vCaYx%+$W4@TSFVa8Zi29U;1}|`}#1mtJ$ef$I{82i>iMm8k_pVas~nSPo5hl zr~G===(%=9=en~d=4oT|d&6^B%GVyJ&Q91q(|cTU;iSC-ovo0&mLYOdZct^FY5m@n zrj+i4g4@a__KjZeXVXlo%FND3CLh~mG?=-CF2?REYxZ%S-%GZv{z_9qA`r`4n~=Zr zP`e=JKo(kZZMd!Md*{w)S!&*=`OE97Y<%<{%&7{uZrxG}XZc4P`jjzE!aFeVA|z;m z`A~A#%#JC^$1kfsI502+#*#YlTsk|+UMkX3H0$e{LP5X(T4p^D6rYr|t6i=agY@BK zOG23o+Ye>Dh0M~FT0!g<-fCifh>!m^Sc;5^LyC$(&81z8S2_{dziyKkq_catTvcM^ zU7}9uZKA61IC)Cj-l@@USrJW@Bgnu%)8%A{A%8J<=$d{}{`{kU-pE)8D#KC{Kri&g z12zv<6X??&xT)3rJ^CnC$>F$-?zChak}%oJ2BHqCm4kQFbx8H^_cukXexWSjn=zbO zOJ>H9K_*?GVzJKI7oVCURyooZh7N708tG8PPZ$v6z{eS!7X*2v`H|-1SlFu_3OA?u z2_TAfqp8Q9`KMH`EoYP~v~echd$ zoLmG+md>?}rRkniSIs@`q%RljMEhd8vzcVAZIeeW;pClZQ)N1KqaW&Vzm=(q=VKD(a3;RC;U`dK(yo`1$$q7k~Mjk2r#*^&N9d zMyhNd=WtavJ2G_aZuvj(Hu<`~^>d76%qp3*_?%EK&a8IIQ>AtT-4$gm{(Uo%;d|j! zsfex&g&cqLxkGN$u_5h2EW6yTw9FyNFQUY;mrlYcjuM%Dq5bv^@395N(pM?3F{GrV zDA&0jSdw`(n)omiS#_Z6->DSPPjC0-le$)Ex_|T@pT5*7`Kb8I)_Foagq`T4PRaH} zn1pSv(#L-`^f;pz33W4URljGN+*BeQ&#cDj7y1+-cm80{tpk_x8CKkDvU4|M`w@zj ze{W%7vvk&S5j@!M?wenc*bN<1`uE~?e|j1bCpFAdGZVuFk9xR%qD?Erj*g+I;zym+ zCk)d3kyW3Je`9u!CDs7rXefi1Eq~}(T{vZve=2gI~%rYV%Ys!*)eOC-mo?v5aEH%0LSsGQ%hnPYRZK0>(>3?axdrtC` zck%>>u`E~!Qx~%$&oy>tZ~yk>femp?D920Fwf*U$v$_-t$*3bSeN^N{#n{);(eZK3 ze>Z9tDZr7|NX}NF+{M2Y??3<@n+xIpavPI)rchi%>oa1I^qE%xGbyF-=jgLD>98Mg z`q8^kPjXZ&nNGV(p2{;r6W zmY245;+a;EAtB$smYU~Js3mKrfmcw_($O_h%dUBOxd4Y-MljI;6e`o<)~ugN-nP1T zagV09Z~LxeRRsUXkvH1o<0Hy_?=j#m`rD%OTqa~(Qq&ifCNFLM_ClW*Bme5Nts?At1Y1?HdYtx7hwyJhob zzc?f5^PoTnhyE|G&B|9OA70SL-(QF09hQb1wcT~0QQMxQw2aHUJ68YY6;tFCTX}<7 zZPs*E0^29iR%%7ZlHCoS!^^^y61~3c#(B;``&-~Hb>}7`U&f$@FU7a(85`+$B=@cS z=BZ&r4>TjEbNLlbJR3YzyoY2F4Mwu&<_l8@FI}L9sNofF#of0m)U`7)jGPSDgh(Z; z9DczlBTfxUQEz*zjJw>*XM58T!~+fQsA3fR3=w>Ley{H;I^e%AsVO7Kp(^I;E8QOB zbW2lF)9UeqI_j5(^4h{BsILfi&ek$5JkG-=7Kb9UcLR5R5TBkL7$S72yXbj!o4SXGB-Aryn@7( zt#vxwK{<1Yq8#-r>@1QrMWs6B$UDzIUhDMx=|<#Ewd%!G39mdj?p9kWrU{%dg8j}WWlE$K< zHC;%W0L90g9J%ldH}sl1YDq5BvvS<$FK7sO610j1kvy}0`kc8EVeRBq;Im%%dGg1z z5g+3>@SsGsX-=CokSJ{YqoZB1W(z23P%3ONQ<|xW%!)GL-X5{DQgUNAe&=a@R&@HI z#xWO7(=t2r{BTZF)*&|jd2P(fKSnW+nM z_48|JmPo)1x#DLMy}I4e3`YlyOE_A$sFDlmnLV5Q;l*udW7Uk;uwdQ+V>wVx$?$P zq9nCJ!Jm$A3(<@iaLzTp;VJMxAeQ1KcaehoUFvWJB}g!!?D>eo$FkALst z$QHpgz+CLoF80!o+dwm&FB~dMy8p10_2UR#Ufy>4og;lq@h89M&p0Kn&rO|Y0WOk7 zod!v?r-)%fp}q-cc~WKhCy6n2JUhY^s`3{MLho6M*S^#ZS>?trS#8;mnK|D&pcTDH z+?{EGZz0`n;~zX5*MZGypZm^>tz{fs@Dpm~o|~A!;;qMURm7-` zYR2ylI%eRM6IT;L7QDUPAzzZ)n;sfIa=2A6W{AthFDjb0(C~cuST`txVXfIj>gZKQlo2U>wxT^fIJ%cJpv@g4K&?~AM7*fjL#Y`8cO z|BnzYRv|K>G?i#3Wd(;UCEv(G7>Eu^=e?bG&(ryv6Mb)R+y{;ew^#Rdz6^oV zoB_7TQct)=AQZh2th$UJ@wLB;!Y1eF;(=-|HiQ`4{63q$ppV_4sJwc(&|gViZs7#8ow((7*c@()5}# zr-GoNLDBDPl5OBmn4p+KKYsP}cBqB>VnPy3l$nIVSpI?Ut&UyHW&Z7X&d&+za@ik~ z8=@b>(-G;KQ&a@zE=6Ltb86Nl$Af|~vd_o_KGzm#f$}d`x)|2_i%%yU3|B3?uxB>@((Rv!`Hml3V zD}wcv;?9f#jp4EJ<>!1cm+enl3&YS3RAQlRxN#;x0GTv)OmZ8X(+V9*+kwkuvPIHjK9hMETwU$ zq{2Pif#=wlT~-Ej@2EdOrdO`MqBcvY6SOJI>%G#kEULO5-obH~30?VjF0&ia_mGh& ziHePo1ZSP!;e>&%%O(?>CME@CR2K9o-jb`Z_MlbNxRC4!QoXKt)g3r zMTP$Fut56+9x4CW4#Q1Z=EDN4)+bGi20sk@hO+_bnS% za{g-ER2JN)@)4^2SctK2y1UqN>lTQHh_X+{&5Wd2IYOZC_H)?(ea1oJ=H8SFXcZJX z;w_6Aot>STbuKd^Ybg7yDY4XnL4t1gJHrQAlu=97 zF%BebFNj0uQ2uUg;-noQ5>>O{W`^UYE=I86Cc@kwgqr5LWUakK$szW5ppTk!@9wx0 zgnep#1k4*l=tLIID@Php160BSzm5;*lLH#&{r~%CBg~LgFk<(-6r`#kFF<-zp7`CHX zYyhV$*N@A5KG}DBnk<5grHccX)^aEV*mW2`HlhKGmHoGe7LWgZ2d+b_eV95>J3}PbFA0W{1I}0qB6g|CMmqfYQP8}4aBAAu z*XMgA>81~zlZ^zUFYKWE&Zufy+wn3(sbvrxeH2IqlWeHOKawH@s4t84Z zO_(4b=^77s$Ec*N93Z9n`6P^=c3+>ZCKCWu-mJCDin-t#`r!kccvaHEP_Vc*SU1|z z0VQ6+d;9tn8?Y?#|9j_GNbp3|e^1!GTgm#re`aBe`F|h5?f%OJS#KXJkbX|Qc`!ny zz{8#w zFKWDOIT)Hb`p;~#hllUvW}^J5ApV0uyCMH9Wi$hCsrRi=Ym z^nCZ`KdAY_X$2O0l!_6Mv=}#wKZg{`!R>aRt}{^z4TEkb?%^-BhSkCMYw}*Vf01Rd zfoDvk*8KS**;K7f!XR)?rXNyG<(o@E*&kLnnEVoaa5DB(^KlP{4Ka%Ne!f!9leG84 zYXM;O10;uM0G$-54>iL_DiAe1p`aH98a^;1uqYB~o>3Pij|D(dET+LGP1QZ-|Jz6q z2I(AbTvb&JWqip`Zsc9QN*|jIAImd-&-IOr9q`3$vgddxg$UJ~Z8esAH|NsEDPFaU zxATEu?~SelUuoB}L)vEhfJVSH)}=G>BI6t$+fm|I0GAZO*KO%E^cGU-;Sf~>yQhgT z8lFfl0Yv_v|6NMc9^b;nfNq8=nczjHJoIr(o0M zocHVWa1_gTKDe1@TN@WWR#3iKq`8qH_*ta%NpAzze}hux5W32RCUbo&-Xf(OP3_Sw-;StOK8BzAv0r*m~ zexiPP&YR%W_0C6v<@PK3SCMMKGHkFLxrdoF+EVxl!M-a3FJK6W6>Cuw2K_eUPOkrA zrR__>sM0@wflD&EK`dZ|Z$vjo_U|gr|49tP)jxJ;6U|!t>dW)ribN%X1jFsoG zZU)Bsl4t2PQq>)>Wv}p%k0N*i8dNC%&+7GKq3?CQmHI8s_uuT@{CGgWQRM$8_|?y) z|0x=o^`G8I)&AT6X@vw2exsG=OvZVPKLoh@t)Nwu{c2)D!r<_VM|J7X4N1GKVRSX} zB(+6`njB#z>C} z0!_#kHk+DGj1V4O(K8@-l|NviG% zUV>rYgSIyN6UjTDH+b~&1TKWUWjB_`VK0`KTPGU=c)u06H{e=EO(v~ZpHurRg4^;) z*qS&1hCMktA|fFqlw9tw_-=V)oYBkYSMaR0pfnUUe8gGbC&mGt%$T~lHH7)e1JKOY zps#=j;WhRB`*($kc!JEAH?D?7_1!(x!szG$G20m_yj8p)hEId7;*DN(h_p~hEJ}tJ ztFs)sY6kRp$&D-K8so4byfJW6BrN4N0BM29^BW=vzgj4q#AoO(*spIQ6tA=KiMC za6s$sc{@?qhuMjVlR=>Ah{!+I{Qn|FJg6%VuC4aN?wOpK2= zd2*LR&u!}S$}fWZE$&a7GvKrT#m3H)%Tx@SzmmHykjM5hP9j@He3@H$_1s{>x;nzBD7Y%1god-kL$%-Q-OZn_1-TVZ_T}oSB zXw08ZN5gWYfr0MsPxw|PZXBBfYuj%7x06d_qyrIr8+3l9!=rLi9upEzaI2-v!OECu zX&Z-6E}u?s$A4#NY$A=I0%@FR04O%Ck&XEGkBjOlARY1uBzRkr2C&b`M@Yh18bneD zf-FEuKIB=wf;<_Bv8=YB5y%c$o3tI1Y&23Y;o8$?%@+#Z?AI$5Z8R#M*S||aRXYP` zjX?kav446I4l3IO78*2B0BcQ5hgE{J0@QX{rt||-v>ncnCk(ADQW#X!1!mXo==eC8 zE;MPV=}}+?s(lO?o;7uE?3Z85z=p+2>g!JeA!g|NT9+%qWG1STFl><*IK{mZ-O};; z8CFY-jSr+|AD7OrHe=R%9n=0f7VC)5qZCFh_I`blYTmA!X#rga(U9ao@>E@Y<&(P% z{peK8^*3wEm&wRr(x{~e2W!La{uUM;KiSoVod)zEb0A8BGb}RWm zRhpmhV^8uDKCH$Ap9F(LCl^u)V? z6`7jcAJZk2=GcFKa;UyTr-K{xo&k^a9=UD#sOiggKE3UhrVghu~M z8%n+Rrj#etas&d3`IHY?Vts~bfvEiJh~6_uCgegIIg3T5}B z-B@y$`IGgA$f_X|8mvw&OPaR!xivS=8&@5fXa;!$G8Y%F|25MdpPk4y>)Z#@vclb%Vai^3TDuQKChjS zfI2K2O?HSEPBd@)M=g*BU>%UAeIqRs*>=l9L|=Vaj#5PwJ4@lQH*cOuL37{>NOhVz}9 z6NeO30gPdoiz=?v*yJQ0!N$i*PO8OyLR9c__neyl0KdkprUc<725-$Z;bYdlM-?La zRB<3*yZ8e1C16F-3rAzY<592Qi(7XX(>UAw(%ce1EQI=m~De1wUJw;z=Lkq5!KY4&( z6XPv&Q)0a_KJ*n;(84ax6DKY2_fW>@#DuR!u})K+qPLIKTa}d&AVfU$A*NpgrMKY0 zK4@Y59@}I`huXduL95Ie5(kzb4>86)a+Vz^9U@z-Lq|tLv)(X+DgF`+JQ^=IKDm>N zA6mog{utupqrfjJG#QR(PsxnrXa8za^8MuUMBuym4RftkNNBG zY#>8Y2YF9r@mxm+r)L*`%QT2AH$N*Eq3R$J9v)r|ldPW$M*M3Fxffc=veHgs*XHfz zb@(wSCnsREOxXdw@w1eQpiQ{nN~AfA?;ZjDow1ZBtrpF4GTs3V2AP(+UM2PF~hgNp|KX@@7+5YTUfXT_a#=q z_+`F0(R74>4M)i}&bkGCvDym(o}a8E3D&}&Af9%be?y#dgt8Ew3t`{jRfHF*(87~| z>`CTwHJhD*Fbmk80!urNMI}Wkt?jCB>c9%m(=vs$TQ?2UH*I!dfzsEQP;`M#!tVK1 zD?q#ZZ?=4;0HDNzfLkDdh}ElN%Rzz(FW;(uakSBQc6OF%ySiJTivu?$!|EIu{k*UZ zK#@=u3+{Y3`ud#}<@>Z#9e=y0YoONoX=N#zSsFNzENx0pn8^cC(T1lg$IWzr9-v2g z*t-MZCCS|39@_>^1T+9%a+G3|RNG=O{lU-LXA^BC9h6+U&lmgq`|Y28p8E-MZ zaJ-EF#-E||YiE+my3yIdC4Bk%t=zQvcmQ#jNnzqQdC-z)-$fqNoLTcF4#{WK* z8Ge3JPzDASdUu>H8K6Ouaihmd_~>7)b?Byen*Ifz*gN;}IfrAMy&#njjl-|Xl$|M@>qugNq=?!)u8 zkXY6Cb4Xl$rQ(?JiR2;Ak}S1OJLL{EGjZ5r7Z-9uAG8A|A~}t*dGV;S2Nk}Bg^)isziI<^t}z_t3rf z0YnGlZg6>iIq~4o(257OXw8H!C_+TMQQRZ}n6lw#seW@7_Rl>v-4^JB315g*>Od<+ z2fkhK?#*!$UU#rc0Q|VY0=ufF>Q7EiijExKCMwyL+J>^=4neQR)5aBm1x|vJW( zFcw@5cZbSlPzA#TYqt!|e^=*8sW)nY0-yA#PHKUkonBCtBnTaNbC7sXVZ0XUUYjPY z2o5v>POAY-Y3Q?=ZKWR7_1jLRP9`h=3AA~wLAOI(rMVnQ%S5OEEu$ zPG!wJuc?DXYYpzrFHQT5JJ$LM9Q-!~)aNxZ+)v}nPX@m2v6&SAz!(uP(BZ!9s@>L* zgIAi@=1K#}%In+PrbHGg?4AeR{DE=o#*A^e`YQQcMo=Am#26H8)ww2j?mGSQW<#_@ z#A_Q*PES3>LawXn*NIbt4unUBHA<+{adtjaMe&w7O5HPnqj4gqkqqBq9j#R>f7 zq%5L>Q=jp_$gf-nNbCy69srdv{t0S}M))YU4Jd!3SD6Sx`lo%fDFLZLz8{Q2TlupZ88g63$BoW=R4U1Lbnp z03{Q0Pc`EXj;`&A^=e;y{p& zXfmQ+4CX4v{b)!Pu12~)U~w6-TD=cazyilfs=`Dzv-(&z6OBWKzL+JcMA}r)R755oNZ%%MBfZjM$ zlrXNz9^)ZO!4b${BTO~8;Z1pg^WO)Ri9cOWXKx^G8}Xt85x z_qDuD*c}9x8p|%(LXa`NV?{iIlB7>&R%t7> zXrbp~J?ET>3E$Kix@F5VQtuX}>epjZFq_}EW-3aJ3&elhl;UEU7hiC><)Wy0B;&?> zHQBo@UQBeCD(*co{8}9u8F>nuUHxV! za=nfDQYAQCK+>VVGCqGenFuohr2C|1*j{+fb zza{sL01C#1x8_7zn_;%?lGK3Nc|e*R7a*-ku?DY)k+T1~P`J?CnISLjH=4a8vB<~Ntz{2jrsCY5KSoJ$L#g$pQdu9JgQH@Fqlb`d+)3)amSM@Ci zi|0ut1W0a;gn7-ET-0vdMo|LqUx|?E)Mhj#Z0yE7q2`(S@%WoaIlkiEucf;-wH2R^ z#eQhsdWEy!Kp0wW{4~w~PHu5VcOg0ryH!G37qD~2ut4%ZgG?s+|I5Xa3N2-vU~9tK75eKOBEOL_!t0nZ5*Zk$UYOpAo0v=5@{dRAm&99LKj16+q$zPk z6SY*6ny^73uF?Rg3tpEI{Qse7*&C~k4q88OQ$X$t{x>xe8u!fp%&vK-@Vp;vI8T;J zYTVyRt#{%oLG>R~~5wVk?un|6VUEpI|(?VlDdhb7EpbaPRD>sWp*ePX(5{+bmdX*&VgkEoZKV zD6Uvd;SiT^_(8=BZK6(pTUGsQezS!M5nqFt)AOMcHb!|eJcxxW>fwAWS>)J#Q>7`V zsw~}^GVd9nZNY09Eiy42Sn)V-_|%O~YWOsYXJgg?PJqAX7YN|z9hnA+7ao9t75|g@ zcP+IuFKz_3X2Ne#Q9v}OXa;0Sj7y*Nm+s6}a%j3_!Df^4Tz%>~d}yLHB6tWD;1&0P za|VG*;*r~4KM_c>a&D4>*FjLn5_98U2GXydZ^YALYL;db6I*uvZDtWvG%5dxBL0`4 zYMr$vW;oBAJzUiES@T_g*-f}mVmsseL0&$XwU%jAtP}he$obGM9cYQy5eqmNBP^J5 zVv=*~@d_iO|F(BNa-ybjxS$HKEZZG~!mjNpvqxq(!?Ko>_~U8}-iy5J*l zd((z=++ZVb-16v`k(*47ix;rNrahU6Ai;_=S!!{obQqGqm}Cc@lQ zf7h~0#^fz*IXf)LTrJCh+a}y_3bk8zg`e5{XKAz@C%{v?m#+C20n|4p9<4wXl4HL; z%yfP0<&at_an2>_NLQ7iaj@5-A+=cZ%a)a2wLX>DGXJ(^=0F%Odn)>*~d^ zS|BCT;huZ-I;iFO*EKsSF^y+5|JG_YNI&C*C(S#a%w<-7x=CgqVm#VrQ+?@rbuh22 z`uhQglALGAugTR-6BHPv@>_rj`2^Yts*JNdfNuYz*%=(*J1x@G!ZKHtq#Y)OB0 zP~qk1dayc305$w;y|DE2^Xn}mB&pH5DKwBh^7cLyU^CL;OAP@p4isD&+I;BN!qo8` zOs5wy;?1~@KfJO|Vc4QusVE5C|1h6itfPPVmCfa5J(5^*pDnFAb@$SQDt@sVnUYWA z!~v(CQv-wNH6aF+9;5tPXGhl_A894?)egr#zY66l^V09=45bIekpq8%Y%^sCBA6(C1kw zA^*1PS1Z3y+D;u-Q3qMRV%<9QYeToA%Hb}v!ndL$j@5@yhD`}kazL6^21)R7v|a4PK$GA}sF|su zfn>l`C|{El1xLqBEit*Co*q+qQK9nepB?_d16V4PdWj6O}I>ym5n@cQQm_R_k-~{24qH24u*PM!mY>L z1zA6nmmew-O!UN%6TH>Fk0JWw2|1xf&9Z0`kchG_I@D|ctXXKW2x7;2^_N*t{H(vf zUd#V9^rsG3U1>VHr5#ehWpC^4?VUQkQJh)TWjo`ufMDC>4h@b{y@ehFBIL>T36uBW3*Ak3Qt*;Oo6N>t2CW_KM=L6< z-IEbN$@Z^0XDKn*>;W8FDUT}G17rPn3l8^9Ky#P`+Ix=L!qp)GHcOGzS497<@72Cz zQB(W`Oz_A=!^`E%OjZalB*dF)UR_?{3Z%5cKD zt8)L~pn3e03yu*1C`50-ak0!SR>{3qeL5gl4+i`6oIpaZY#(j`GpvTBVvz{wufl+e zr2YEPKuYnxr~a|TW~+BJ-|+|D-Kv1RolL=d15JrYuU12puCDIST&@A%8Y;d59p2F0 zP4reMY}UEPVUB-W$13=7?7P3WQk|`R6L)^CH0@}@9D+ou>7BL*D;rg!^6O#iw30QE2LkNK{hR zb2@=`tarH_f-hC-Xhk1SdCz5czL%(E=;SjoNvNK8`;Ia)GID~CY&Y$vohDZ_7%M3^ z5lk2j`m^L3Gxjwq`?@wS<=JPvL&)#cAflc>u9U`k)_&k&dcyW-Z-b#AI1V0HtLb-_ zo0|@M(NF!kv*CDn`sYAZ-=(wYA%43yDXO@+a-r%x%ZKVxQZYTMEP}OvDoM{-yw^Cs z@Q`Qq#>LgPk0bK5zM$(J;sYBC3+*C%CCz^rH6S0^FE#NpB`p*S>>A%nRhODm-;dL- zLHI1a4rHdhulmx<{zYOH240)?Njfj&;OuNC@*bsy!XaY&Kt?9*xYD|GuD6d z#A3_cMCR+aE=7cd*YB-w9|XL^X?EhAchvY5ZTgyiA22DK(KWpn{KQ`_$8nH&6%DIh4Cr_4~GX0x~7=?lypm83E_?wq` z#H0lG)UDvpZh!Kbn#cl9cIMFNX$|IGxWjfK&wwYIT|BH_|JM0VEG9j7htBrOP z-qRYY3Jd;!0@I1Kk5iA!KMu9*wLMKssfIb6=TUZv_BJXr&2Bdmuq7S-fY;j&ykm#F zCX%tC(>dcy|GqD2FJE&!DEyW3H6K%UAVdga$vwZLo}M2hI$Bvng)Av4Vf-${VYbOQ zLTk;*toiPRS(YQ7Ia~bXY=id@m%g}TJ1KWTjcefCosjp;3%JsWuQ^r-E+qD&<>UdLCdevl%SpRZYSB<7d@+a~C3cX-@`rvlvN7g)3Rk;(r zkjyOELt(PkO6YKy1uOK??H_T-9h{n>x?j4VVL0ClL<1_SzM7-AWOu*PNZcC` ztb>J{qiC{&%@veRz_;o$pv7Is)kB-!nBUiLIpsr_DpVhaBQ?fHsG;@biG8sP4>QJ; z@Yi3KN9qjg;mMDFGeEZK)|;iJrM0eGuL#s;pYx6-TN~<47e(-uv|F;nhet=4Sq0UT zUkGPrCn@F4efV=|3W?@^_`_pr*vp@uvu^aJh5Th_uCA7rRtK6yZkKdv9KmMkyv8Ck zYSsWg#A^kW)8!70y@I(f67!15+l)68h_r~`$qWT7J?nSTQ5V_l!6_-qtSWd1HqF&g zcj5bu@$wl8NHN`dx8-C$KxQHXMP5T)F+DxKjW~2|5<>^Vw9di<-PGInC_}T<9x5Z% zw5gJo4Xa=OvEe^xY&ye)`WTOO%`^SeTVLAj3%|J+Jm{qWUX>GSA7MCuuBsYWMJ{kL z)KezUe_%{!bv9KeWuQQe@luvXwmR`1YfEp3^>x64dS&5z1qAy8uRD?T!SyHO!Dst5 zmJP0qFFtxcKW^^LCIY8ESc~y!jo?}h6%`t0)jr)iOGY2KHUt@2-Nxts!i9P_`^mbj z=`o$>GF%_o+OFV1HG_Az=JHN46%Fe;`79Yj4+O0K4ARs*>RXPLYFc?`OsuVPiA_iC zRU`>MJ(l%71N9Wdq4DwYHCX$lRA=aWYhQj{&jXb>ZW!csyFaC zxI9mJJ5lDaR4LQ{&$7%*a!jq3$&_W|+!2QlwY5jXPz~R@-Kq;h*vQ_FPf)vIV`D=E z`YvE%prfPXl^n&uA0FT}+)f;!E50AWpXk0DM!Vv;dn+ql-LE`~DagfrVL0&zDm9Ae zNdw^*)e70YfcGsQ(>;DLaXLQq{qgfX?I8hV z>}`v0i4kt@H2ujkH?a)dAZ0GQ%obSisqMPcIPvG(`fQpH${nJ3-PAIxw%v6u3%4xd zU#=INo}Q)<{$6FUjs(S_k>qZ)%qN#~VMasf@9~&`YRo#Bo%85ht_ZUl3;N>h`|sag z!Ko04bYCvz_HdEP{`K78UMpE5g*^%VT$`#ob|Z0i24Zs!#I+?$6AD=YJc0^M|BzTO zzWzNb<6BbaR2Z^v3mKq}uwv^~ZbfS7ud{<3a)_nt*Y*d3RCUI@6KUn=^@-1CoPZq@{mDf#L*#Gi@x8R`y1w5stmTIFdN0b{(iy=IT-#^ zMo`bxNtr0?HDf>RL#uM86p{U{8!}=2@%Is1vi|;j&4|(b6Rr0WUN%9Qcvrt7y4%Nm zD1z>^l5{x)7Ofai+(+T%*JrteM078~Atsy;K_d@@+Tnz^Y?1EFG`uD%#nB(^sfkFn zY7;XOsebC0od%xwGDE_c&aM(Ptj7_urLd!e`og#7jUTUWxhPE5xtQY(`Wj!JJDYv% z@D~@l#>>O&2n@Xw{+uZ`H3L~qHQW(1An0*c=`yn>cu)!dRRL@&8M@;f!xLCP+4{x_}4hQ{nDwlYtna0nW zrp74)xkvOG5NB5ZO<*UWQ*uCWIgp1&SMi=2XI#6c0(i zwQG)(_N?n=PJgGY;S&((gS`=S{HQ>OD3o5<<}!uS+xk6BjVs~Gdv)VzGeptGbO8P3 zjf(^p?gH!gBEsNU%B%h3H~SOnb(F->aXHUvA2+{-J-fcuYna&*NS`KyGeXC&*PW|t z;roE1y!V)~^!d=o+769wYX$EUzw4>GAYS+&3)|D~-20V=)vLq`Q)0vap0Iuw8P`Cx zPf+wByv*)V1t>n9XUY`wQ3<*I$AdJOxKIvKg_8#;KY|P%#i!08eicWaBpLTGT~sNu zC7YpJC^`hhJ!96dOYUc#r^)a8h1P{N!WPR}YOy<%8d~5mXUI5v8FKFVpx9>ODvzKb zjxJ3z+*Di5EOT;n6ceh|3ZEp&qNw=sL%a0TCwH;XWnsk%O<=j2>B4`6)crFB8_Fv9 zhyS0}uKXYB^^1QjA!O`Zu4NQqLdGC7uBAa)B0_eTMlRX1XB#pZ35hV4k`Q4mWv@Y$ zD<50-#)L$eXi+LgzR$gX!}s&+^TT;Pud}_+`Ml40USZYKI8mIp@bug1+gS&55%P$J z&qJ9`8*-mm!RDPETv+$khp>CRTx^@K&Bn*#=K8?I-7wR2BVQ$>9unW39f?)Fvvs37 zbMpy9x}ITOw0A`#4ymZ{ws%FJqToJ{uBo@a{YO5ognh63^Y_-Y8&uv9G`L+k{WdWq zjsqtSe2#lTm1E2aoB4n_bjk4s$WqdW56fNjt)u&FT&!Vs^acb3j2j$jwcIKXVKdT` zq45R=$cM)e02SuLh{aYFx55=L7fQfu_ZoeB1k%zHlgZ2ob}Ld5QL@rg%|6W{k@c~n z9fmGxtIsc>Rt6Gow8KB&-;C_>`kVzh`rX7mtg1Tp=!6Z=d{bboo)9AdSws#RS}p^I zkXUS`lrO37VugQD;kULp`-jA+jwN?@n6x4YLFq5NU)t!2Azd@%3=@dVjx9DBfB6Js zoUGjDi>;IOrVqx90NC1qS;3ktrMqOMm)<8f0?u`+j%>C;-J!)F##(mVB)nIy(pxkp z3RF^pZywg&)SQmEmr5ynf!9`-NC&IE^gBR^4)PbizBb*{CVKoC?zkIZFD^l=xItw^ zw`WA~!K3(=bUZSZi#^E+;@8X`f$e=73qHiM7oGkFt#ro5#>lbN23ET@+avDd1(2X5_O`hvAyjy>9}9>9;D=PJ!gwI1Q;x(3uj$!UCk z00ICZA7)+R7^}mFN1()abCNDAXdywy=z-DJ0%0e%M4FqM^LJvR)5(C{K)zc zV*Itz0gy>?@#t-9EtfV(UPi!>b3vM842H*&lasN_=g%!(A`l2MAeRXUIY2BnsKw zmh>{y#uOX*q%_Vu}c=bW6J5U)UpgA`O2=t*K z&P;}2-+=OEa%Lu4hZs=MF3NRQX|r4-*RHFox;h4ZY9s25^Eq%^;k4obIE?L^=G;U& z9A}S{-5(9XntaSZedLM`+9pZ2*ZmAqS_)v9&!u%wNWF_9X;rzDdd^Y9UPg=t#mWQs z_v`XWsZI0z91MvMx=fcf#V2yzyq7r1FE7i_^{MvJqp!e#F$m}F&(`&`j!2dSo z*G@F<_bu~{4It)gyf00#TZ66m_S)}qTwx)UCzk3FiE(!LPZjgL-8I#1_q4Vc2M!!S zI664|ZAfw?4+9h-iY(+Q6@44TtJxD;ERCQub8U}J^fuK#cAMUlB3hEGG{@uD<&E*J zSnwJFnj5XmhnXUR;7YvHu>; z=5}w57TzG$Edf7GA_MY?uWyhs-O)aS%4|`s&}mLQWMseY0}<^W?J@s2TJXzNd0Ff6 zAnacAc9|orhPj!On#o{6LNptdVCaEmIZzU(wljZK&Tmlg3|hzzH6~5VhQn! z>~F3g>Rk+RSRd6@z<Eh*mpu_8~6Yg){X{)95@x29-+vZC;5*Mvh33YDxm> z%lPxD{Kbk&DTq3Ii#_0fzH4{Yn*w*;Ae)$QGlv$J0azCVoRi=hGgwbCb*LdQ_K1)H zdv`u!i}d-QxWOhfM<=Ix`nV~u5zkz^tX@_oX4W4q#2BBOi>v=gSeK)z0gp{!{Tz3n zE;*2ug%W(O?8|knr!m%*_7{EICT;ax_>gYWLFhZQ86tDZrxL5UouR=eF&g>kZ%}W` zd97qqHq!;2%P8bmK8-WiVmmd3)A)T>B-p-vE%lu3J6hIHUTrBXEEJ<`-kdR{9e3-V z=6>lQJZA8D^>mOz(}Y&VMDX@Djt?@qKRRGhJq^jcH|}aWz)?`5&-lf#iZUzjpC;9~ z9}|_X%`If}vaQ@jru5&2KR#(`ag11zvi2{T!W9^p94mSP{ZxedLyMPzpixC-;77T6 zc`pCjdPe02>f9CEKYnd6se|mnawuXdFuT$Yg?h; zIcQQ11B~Mf@Nu%AfV#}KKP4*l{r<^r1d5Qw-+%-%HOw;7mUeJ$4T{er59`jgTJ0Vt zO63s@?A+ZM4bwFwjb*n*e+iJ2>@2K}=u&@vWm&#Yw#I8$w71Vz*VNEEGz=wqN!O26 zh|BBF!x)a}^C}qkv&c>9D=i_-z~f9|8TOEP2~RH-R)Cn$q-!2#>2B=i?582nYyC5W z{1TEpSlUY4B;eJE`4B5gi^95hCPyi7I6O5a>IyBSx%2y!DqgJifT2@x}Z^LB7XJrs_<6OXYG#%O{lR<+|0yTKm`8sw+7HB7Vp&IwGrJ+ zD)Z0$HH=yI>2gE#KAAl1v$D0oaH$_5`^loa4la~{hE0pioNDC5K?!Qfzo--9V%Iwn zp~Xyt&O)PdT2GH{+wh_5oh<6SS0-VW5OFz=a6IFMx7;P>nADh`_UOcfHb`a)epqOW zA5v!H3D@!#AzH&vDP)}r0S`tWd>D#*4j9E1Q=Hd&CP9mm&c7#(I#9&KV-tN8mk3lzlaCQ9aU8uq1JDz|j?I(@!S*eGsY2f%J!G!t6KLpn`U= z^sAgU)dyp9?K~5U_1ilJnrgYGBXr*{FUKeRU_=GK9-F>=&coxbr>*U+ez+$uFAst^ z*^>hE2pC-q=ID=zmk$JYEqL<b8 ds{g-^_7P$u?vIfMk0&AUoiRO&r + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + From b9bde125f07661f27293922e201ac94b2d78ea20 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds <40469975+dylanmcreynolds@users.noreply.github.com> Date: Wed, 6 Jul 2022 09:03:31 +0200 Subject: [PATCH 48/98] fix sphynx build --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 48a8171..28b3c36 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -90,7 +90,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From 50d338f906d577045860e15fe0982ae443467572 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Sun, 7 Aug 2022 14:49:27 -0700 Subject: [PATCH 49/98] fix merge --- pyscicat/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 5978e2e..d71dcf4 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -346,8 +346,6 @@ def update_dataset(self, dataset: Dataset, pid) -> str: logger.info(f"dataset updated {pid}") return pid - upsert_derived_datasets = datasets_derived_upsert - def datasets_datablock_create( self, datablock: Datablock, datasetType: str = "RawDatasets" ): From 51840a4bb053b1de622d45c3bae6594ef9b501a1 Mon Sep 17 00:00:00 2001 From: Max Novelli Date: Wed, 10 Aug 2022 11:58:50 +0200 Subject: [PATCH 50/98] Update model.py modify according comments in PR --- pyscicat/model.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyscicat/model.py b/pyscicat/model.py index a760906..5430128 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -13,7 +13,7 @@ class DatasetType(str, enum.Enum): derived = "derived" -class Ownable(BaseModel): +class Ownable(MongoQueryable): """Many objects in SciCat are ownable""" ownerGroup: str @@ -40,7 +40,7 @@ class User(BaseModel): id: str -class Proposal(Ownable, MongoQueryable): +class Proposal(Ownable): """ Defines the purpose of an experiment and links an experiment to principal investigator and main proposer """ @@ -62,7 +62,7 @@ class Proposal(Ownable, MongoQueryable): ] # may need updating with the measurement period model -class Sample(Ownable, MongoQueryable): +class Sample(Ownable): """ Models describing the characteristics of the samples to be investigated. Raw datasets should be linked to such sample definitions. @@ -105,7 +105,7 @@ class Instrument(MongoQueryable): customMetadata: Optional[dict] -class Dataset(Ownable, MongoQueryable): +class Dataset(Ownable): """ A dataset in SciCat, base class for derived and raw datasets """ @@ -183,7 +183,7 @@ class DataFile(MongoQueryable): perm: Optional[str] = None -class Datablock(Ownable, MongoQueryable): +class Datablock(Ownable): """ A Datablock maps between a Dataset and contains DataFiles """ @@ -199,7 +199,7 @@ class Datablock(Ownable, MongoQueryable): datasetId: str -class OrigDatablock(Ownable, MongoQueryable): +class OrigDatablock(Ownable): """ An Original Datablock maps between a Dataset and contains DataFiles """ From b8b06f82ca80f68a216ecca7e812faf910b44d16 Mon Sep 17 00:00:00 2001 From: Max Novelli Date: Wed, 10 Aug 2022 12:09:32 +0200 Subject: [PATCH 51/98] Update model.py switch order of MongoQueryable and Ownable --- pyscicat/model.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyscicat/model.py b/pyscicat/model.py index 5430128..85c1e3f 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -13,13 +13,6 @@ class DatasetType(str, enum.Enum): derived = "derived" -class Ownable(MongoQueryable): - """Many objects in SciCat are ownable""" - - ownerGroup: str - accessGroups: List[str] - - class MongoQueryable(BaseModel): """Many objects in SciCat are mongo queryable""" @@ -29,6 +22,13 @@ class MongoQueryable(BaseModel): createdAt: Optional[str] +class Ownable(MongoQueryable): + """Many objects in SciCat are ownable""" + + ownerGroup: str + accessGroups: List[str] + + class User(BaseModel): """Base user.""" From 5a182945d8caeaa9e143e48b92b7ecb015ac21a9 Mon Sep 17 00:00:00 2001 From: Dylan McReynolds Date: Wed, 10 Aug 2022 09:07:24 -0700 Subject: [PATCH 52/98] REL: v0.2.3 From 985e857135457d54c82d4545ac938eecf0e14549 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 17 Aug 2022 16:23:15 +0200 Subject: [PATCH 53/98] Remove code duplication in client --- pyscicat/client.py | 433 +++++++++++++-------------------- pyscicat/tests/test_client.py | 67 +++++ pyscicat/tests/test_suite_2.py | 12 +- 3 files changed, 236 insertions(+), 276 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index d71dcf4..b764960 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -5,9 +5,11 @@ import hashlib import logging import json -from typing import List -import urllib +import re +from typing import Optional +from urllib.parse import urljoin, quote_plus +from pydantic import BaseModel import requests from pyscicat.model import ( @@ -17,7 +19,6 @@ OrigDatablock, RawDataset, DerivedDataset, - PublishedData, ) logger = logging.getLogger("splash_ingest") @@ -48,7 +49,7 @@ class ScicatClient: def __init__( self, - base_url: str = None, + base_url: str, token: str = False, username: str = None, password: str = None, @@ -78,7 +79,6 @@ def __init__( self._password = password # default password self._token = token # store token here self._headers = {} # store headers - assert self._base_url is not None, "SciCat database URL must be provided" logger.info(f"Starting ingestor talking to scicat at: {self._base_url}") @@ -89,68 +89,47 @@ def __init__( self._token = get_token(self._base_url, self._username, self._password) self._headers["Authorization"] = "Bearer {}".format(self._token) - def _send_to_scicat(self, url, dataDict=None, cmd="post"): + def _send_to_scicat(self, cmd: str, endpoint: str, data: BaseModel = None): """sends a command to the SciCat API server using url and token, returns the response JSON Get token with the getToken method""" - if cmd == "post": - response = requests.post( - url, - params={"access_token": self._token}, - headers=self._headers, - json=dataDict, - timeout=self._timeout_seconds, - stream=False, - verify=True, - ) - elif cmd == "delete": - response = requests.delete( - url, - params={"access_token": self._token}, - headers=self._headers, - timeout=self._timeout_seconds, - stream=False, - ) - elif cmd == "get": - response = requests.get( - url, - params={"access_token": self._token}, - headers=self._headers, - json=dataDict, - timeout=self._timeout_seconds, - stream=False, - ) - elif cmd == "patch": - response = requests.patch( - url, - params={"access_token": self._token}, - headers=self._headers, - json=dataDict, - timeout=self._timeout_seconds, - stream=False, - ) - return response - - # Future support for samples - # def upload_sample(self, sample): - # sample = { - # "sampleId": projected_start_doc.get('sample_id'), - # "owner": projected_start_doc.get('pi_name'), - # "description": projected_start_doc.get('sample_name'), - # "createdAt": datetime.isoformat(datetime.utcnow()) + "Z", - # "sampleCharacteristics": {}, - # "isPublished": False, - # "ownerGroup": owner_group, - # "accessGroups": access_groups, - # "createdBy": self._username, - # "updatedBy": self._username, - # "updatedAt": datetime.isoformat(datetime.utcnow()) + "Z" - # } - # sample_url = f'{self._base_url}Samples' - - # resp = self._send_to_scicat(sample_url, sample) - # if not resp.ok: # can happen if sample id is a duplicate, but we can't tell that from the response - # err = resp.json()["error"] - # raise ScicatCommError(f"Error creating Sample {err}") + return requests.request( + method=cmd, + url=urljoin(self._base_url, endpoint), + json=data.dict(exclude_none=True) if data is not None else None, + params={"access_token": self._token}, + headers=self._headers, + timeout=self._timeout_seconds, + stream=False, + verify=True, + ) + + def _call_endpoint( + self, + cmd: str, + endpoint: str, + data: BaseModel = None, + operation: str = "", + allow_404=False, + ) -> Optional[dict]: + response = self._send_to_scicat(cmd=cmd, endpoint=endpoint, data=data) + result = response.json() + if not response.ok: + err = result.get("error", {}) + if ( + allow_404 + and response.status_code == 404 + and re.match(r"Unknown (.+ )?id", err.get("message", "")) + ): + # The operation failed but because the object does not exist in SciCat. + logger.error("Error in operation %s: %s", operation, err) + return None + raise ScicatCommError(f"Error in operation {operation}: {err}") + logger.info( + "Operation '%s' successful%s", + operation, + f"pid={result['pid']}" if "pid" in result else "", + ) + return result def datasets_replace(self, dataset: Dataset) -> str: """ @@ -173,20 +152,16 @@ def datasets_replace(self, dataset: Dataset) -> str: """ if isinstance(dataset, RawDataset): - dataset_url = self._base_url + "RawDataSets/replaceOrCreate" + dataset_url = "RawDataSets/replaceOrCreate" elif isinstance(dataset, DerivedDataset): - dataset_url = self._base_url + "DerivedDatasets/replaceOrCreate" + dataset_url = "DerivedDatasets/replaceOrCreate" else: - logging.error( + raise TypeError( "Dataset type not recognized (not Derived or Raw dataset instances)" ) - resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error creating dataset {err}") - new_pid = resp.json().get("pid") - logger.info(f"new dataset created {new_pid}") - return new_pid + return self._call_endpoint( + cmd="post", endpoint=dataset_url, data=dataset, operation="datasets_replace" + ).get("pid") """ Upload or create a new dataset @@ -210,25 +185,17 @@ def datasets_create(self, dataset: Dataset) -> str: Returns ------- - dataset : Dataset - Dataset created including the pid (or unique identifier) of the newly created dataset + str + pid of the dataset Raises ------ ScicatCommError Raises if a non-20x message is returned """ - dataset_url = self._base_url + "Datasets" - resp = self._send_to_scicat(dataset_url, dataset.dict(exclude_none=True)) - - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error creating dataset {err}") - - new_pid = resp.json().get("pid") - logger.info(f"new dataset created {new_pid}") - - return resp.json() + return self._call_endpoint( + cmd="post", endpoint="Datasets", data=dataset, operation="datasets_create" + ).get("pid") """ Upload a new dataset @@ -260,14 +227,12 @@ def datasets_raw_replace(self, dataset: Dataset) -> str: ScicatCommError Raises if a non-20x message is returned """ - raw_dataset_url = self._base_url + "RawDataSets/replaceOrCreate" - resp = self._send_to_scicat(raw_dataset_url, dataset.dict(exclude_none=True)) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error creating raw dataset {err}") - new_pid = resp.json().get("pid") - logger.info(f"new dataset created {new_pid}") - return new_pid + return self._call_endpoint( + cmd="post", + endpoint="RawDataSets/replaceOrCreate", + data=dataset, + operation="datasets_raw_replace", + ).get("pid") """ Upload a raw dataset @@ -299,18 +264,14 @@ def datasets_derived_replace(self, dataset: Dataset) -> str: ScicatCommError Raises if a non-20x message is returned """ - derived_dataset_url = self._base_url + "DerivedDataSets/replaceOrCreate" - resp = self._send_to_scicat( - derived_dataset_url, dataset.dict(exclude_none=True) - ) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error creating raw dataset {err}") - new_pid = resp.json().get("pid") - logger.info(f"new dataset created {new_pid}") - return new_pid - - def update_dataset(self, dataset: Dataset, pid) -> str: + return self._call_endpoint( + cmd="post", + endpoint="DerivedDataSets/replaceOrCreate", + data=dataset, + operation="datasets_derived_replace", + ).get("pid") + + def update_dataset(self, dataset: Dataset, pid: str) -> str: """Updates an existing dataset Parameters @@ -330,25 +291,16 @@ def update_dataset(self, dataset: Dataset, pid) -> str: ScicatCommError Raises if a non-20x message is returned """ - if pid: - encoded_pid = urllib.parse.quote_plus(pid) - endpoint = "Datasets/{}".format(encoded_pid) - url = self._base_url + endpoint - else: - logger.error("No pid given. You must specify a dataset pid.") - return None - - resp = self._send_to_scicat(url, dataset.dict(exclude_none=True), cmd="patch") - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error updating dataset {err}") - pid = resp.json().get("pid") - logger.info(f"dataset updated {pid}") - return pid + return self._call_endpoint( + cmd="patch", + endpoint=f"Datasets/{quote_plus(pid)}", + data=dataset, + operation="update_dataset", + ).get("pid") def datasets_datablock_create( self, datablock: Datablock, datasetType: str = "RawDatasets" - ): + ) -> dict: """ Create a new datablock for a dataset. The dataset can be both Raw or Derived. @@ -372,16 +324,13 @@ def datasets_datablock_create( ScicatCommError Raises if a non-20x message is returned """ - url = ( - self._base_url - + f"{datasetType}/{urllib.parse.quote_plus(datablock.datasetId)}/origdatablocks" + endpoint = f"{datasetType}/{quote_plus(datablock.datasetId)}/origdatablocks" + return self._call_endpoint( + cmd="post", + endpoint=endpoint, + data=datablock, + operation="datasets_datablock_create", ) - resp = self._send_to_scicat(url, datablock.dict(exclude_none=True)) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error creating datablock. {err}") - - return resp.json() """ Upload a Datablock @@ -413,16 +362,13 @@ def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: Raises if a non-20x message is returned """ - encoded_pid = urllib.parse.quote_plus(origdatablock.datasetId) - endpoint = "Datasets/" + encoded_pid + "/origdatablocks" - url = self._base_url + endpoint - - resp = self._send_to_scicat(url, origdatablock.dict(exclude_none=True)) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error creating dataset original datablock. {err}") - - return resp.json() + endpoint = f"Datasets/{quote_plus(origdatablock.datasetId)}/origdatablocks" + return self._call_endpoint( + cmd="post", + endpoint=endpoint, + data=origdatablock, + operation="datasets_origdatablock_create", + ) """ Create a new SciCat Dataset OrigDatablock @@ -433,7 +379,7 @@ def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: def datasets_attachment_create( self, attachment: Attachment, datasetType: str = "RawDatasets" - ): + ) -> dict: """ Create a new Attachment for a dataset. Note that datasetType can be provided to determine the type of dataset @@ -454,22 +400,13 @@ def datasets_attachment_create( ScicatCommError Raises if a non-20x message is returned """ - url = ( - self._base_url - + f"{datasetType}/{urllib.parse.quote_plus(attachment.datasetId)}/attachments" - ) - logging.debug(url) - resp = requests.post( - url, - params={"access_token": self._token}, - timeout=self._timeout_seconds, - stream=False, - json=attachment.dict(exclude_none=True), - verify=True, + endpoint = f"{datasetType}/{quote_plus(attachment.datasetId)}/attachments" + return self._call_endpoint( + cmd="post", + endpoint=endpoint, + data=attachment, + operation="datasets_attachment_create", ) - if not resp.ok: - err = resp.json()["error"] - raise ScicatCommError(f"Error uploading thumbnail. {err}") """ Create a new attachement for a dataset @@ -478,7 +415,9 @@ def datasets_attachment_create( upload_attachment = datasets_attachment_create create_dataset_attachment = datasets_attachment_create - def datasets_find(self, skip=0, limit=25, query_fields=None): + def datasets_find( + self, skip: int = 0, limit: int = 25, query_fields: Optional[dict] = None + ) -> Optional[dict]: """ Gets datasets using the fullQuery mechanism of SciCat. This is appropriate for cases where might want paging and cases where you want to perform @@ -511,13 +450,12 @@ def datasets_find(self, skip=0, limit=25, query_fields=None): query_fields = json.dumps(query_fields) query = f'fields={query_fields}&limits={{"skip":{skip},"limit":{limit},"order":"creationTime:desc"}}' - url = f"{self._base_url}/Datasets/fullquery?{query}" - response = self._send_to_scicat(url, cmd="get") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() + return self._call_endpoint( + cmd="get", + endpoint=f"Datasets/fullquery?{query}", + operation="datasets_find", + allow_404=True, + ) """ find a set of datasets according the full query provided @@ -526,7 +464,7 @@ def datasets_find(self, skip=0, limit=25, query_fields=None): get_datasets_full_query = datasets_find find_datasets_full_query = datasets_find - def datasets_get_many(self, filter_fields=None) -> List[Dataset]: + def datasets_get_many(self, filter_fields: Optional[dict] = None) -> Optional[dict]: """ Gets datasets using the simple fiter mechanism. This is appropriate when you do not require paging or text search, but @@ -550,15 +488,11 @@ def datasets_get_many(self, filter_fields=None) -> List[Dataset]: """ if not filter_fields: filter_fields = {} - filter_fields = json.dumps(filter_fields) - url = f'{self._base_url}/Datasets/?filter={{"where":{filter_fields}}}' - response = self._send_to_scicat(url, cmd="get") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() + endpoint = f'/Datasets/?filter={{"where":{filter_fields}}}' + return self._call_endpoint( + cmd="get", endpoint=endpoint, operation="datasets_get_many", allow_404=True + ) """ find a set of datasets according to the simple filter provided @@ -567,7 +501,7 @@ def datasets_get_many(self, filter_fields=None) -> List[Dataset]: get_datasets = datasets_get_many find_datasets = datasets_get_many - def published_data_get_many(self, filter=None) -> List[PublishedData]: + def published_data_get_many(self, filter=None) -> Optional[dict]: """ retrieve all the published data using the simple fiter mechanism. This is appropriate when you do not require paging or text search, but @@ -585,21 +519,16 @@ def published_data_get_many(self, filter=None) -> List[PublishedData]: filter : dict Dictionary of filtering fields. Must be json serializable. """ - if not filter: - filter = None - else: + if filter: filter = json.dumps(filter) - url = f"{self._base_url}PublishedData" + ( - f'?filter={{"where":{filter}}}' if filter else "" + endpoint = "PublishedData" + (f'?filter={{"where":{filter}}}' if filter else "") + return self._call_endpoint( + cmd="get", + endpoint=endpoint, + operation="published_data_get_many", + allow_404=True, ) - print(url) - response = self._send_to_scicat(url, cmd="get") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() """ find a set of published data according to the simple filter provided @@ -608,7 +537,7 @@ def published_data_get_many(self, filter=None) -> List[PublishedData]: get_published_data = published_data_get_many find_published_data = published_data_get_many - def datasets_get_one(self, pid=None) -> Dataset: + def datasets_get_one(self, pid: str) -> Optional[dict]: """ Gets dataset with the pid provided. This function has been renamed. Provious name has been maintained for backward compatibility. @@ -619,30 +548,16 @@ def datasets_get_one(self, pid=None) -> Dataset: pid : string pid of the dataset requested. """ - - encode_pid = urllib.parse.quote_plus(pid) - url = f"{self._base_url}/Datasets/{encode_pid}" - response = self._send_to_scicat(url, cmd="get") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() + return self._call_endpoint( + cmd="get", + endpoint=f"Datasets/{quote_plus(pid)}", + operation="datasets_get_one", + allow_404=True, + ) get_dataset_by_pid = datasets_get_one - # this method is future, needs testing. - # def update_dataset(self, pid, fields: Dict): - # response = self._send_to_scicat( - # f"{self._base_url}/Datasets", dataDict=fields, cmd="patch" - # ) - # if not response.ok: - # err = response.json()["error"] - # logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - # return None - # return response.json() - - def instruments_get_one(self, pid: str = None, name: str = None) -> dict: + def instruments_get_one(self, pid: str = None, name: str = None) -> Optional[dict]: """ Get an instrument by pid or by name. If pid is provided it takes priority over name. @@ -665,29 +580,23 @@ def instruments_get_one(self, pid: str = None, name: str = None) -> dict: """ if pid: - encoded_pid = urllib.parse.quote_plus(pid) - endpoint = "/Instruments/{}".format(encoded_pid) - url = self._base_url + endpoint + endpoint = f"Instruments/{quote_plus(pid)}" elif name: - endpoint = "/Instruments/findOne" query = json.dumps({"where": {"name": {"like": name}}}) - url = self._base_url + endpoint + "?" + query + endpoint = f"Instruments/findOne?{query}" else: - logger.error( - "Invalid instrument pid and/or name. You must specify instrument pid or name" - ) - return None + raise ValueError("You must specify instrument pid or name") - response = self._send_to_scicat(url, cmd="get") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() + return self._call_endpoint( + cmd="get", + endpoint=endpoint, + operation="instruments_get_one", + allow_404=True, + ) get_instrument = instruments_get_one - def samples_get_one(self, pid: str = None) -> dict: + def samples_get_one(self, pid: str) -> Optional[dict]: """ Get a sample by pid. This function has been renamed. Previous name has been maintained for backward compatibility. @@ -704,20 +613,16 @@ def samples_get_one(self, pid: str = None) -> dict: dict The sample with the requested pid """ - - encoded_pid = urllib.parse.quote_plus(pid) - endpoint = "/Samples/{}".format(encoded_pid) - url = self._base_url + endpoint - response = self._send_to_scicat(url, cmd="get") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() + return self._call_endpoint( + cmd="get", + endpoint=f"Samples/{quote_plus(pid)}", + operation="samples_get_one", + allow_404=True, + ) get_sample = samples_get_one - def proposals_get_one(self, pid: str = None) -> dict: + def proposals_get_one(self, pid: str = None) -> Optional[dict]: """ Get proposal by pid. This function has been renamed. Previous name has been maintained for backward compatibility. @@ -733,19 +638,13 @@ def proposals_get_one(self, pid: str = None) -> dict: dict The proposal with the requested pid """ - - endpoint = "/Proposals/" - url = self._base_url + endpoint + pid - response = self._send_to_scicat(url, cmd="get") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() + return self._call_endpoint( + cmd="get", endpoint=f"Proposals/{quote_plus(pid)}", allow_404=True + ) get_proposal = proposals_get_one - def datasets_origdatablocks_get_one(self, pid: str = None) -> dict: + def datasets_origdatablocks_get_one(self, pid: str) -> Optional[dict]: """ Get dataset orig datablocks by dataset pid. This function has been renamed. Previous name has been maintained for backward compatibility. @@ -761,19 +660,16 @@ def datasets_origdatablocks_get_one(self, pid: str = None) -> dict: dict The orig_datablocks of the dataset with the requested pid """ - - encoded_pid = urllib.parse.quote_plus(pid) - url = f"{self._base_url}/Datasets/{encoded_pid}/origdatablocks" - response = self._send_to_scicat(url, cmd="get") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() + return self._call_endpoint( + cmd="get", + endpoint=f"/Datasets/{quote_plus(pid)}/origdatablocks", + operation="datasets_origdatablocks_get_one", + allow_404=True, + ) get_dataset_origdatablocks = datasets_origdatablocks_get_one - def datasets_delete(self, pid: str = None) -> dict: + def datasets_delete(self, pid: str) -> Optional[dict]: """ Delete dataset by pid This function has been renamed. Previous name has been maintained for backward compatibility. @@ -789,16 +685,12 @@ def datasets_delete(self, pid: str = None) -> dict: dict response from SciCat backend """ - - encoded_pid = urllib.parse.quote_plus(pid) - endpoint = "/Datasets/{}".format(encoded_pid) - url = self._base_url + endpoint - response = self._send_to_scicat(url, cmd="delete") - if not response.ok: - err = response.json()["error"] - logger.error(f'{err["name"]}, {err["statusCode"]}: {err["message"]}') - return None - return response.json() + return self._call_endpoint( + cmd="delete", + endpoint=f"/Datasets/{quote_plus(pid)}", + operation="datasets_delete", + allow_404=True, + ) delete_dataset = datasets_delete @@ -852,9 +744,10 @@ def get_token(base_url, username, password): verify=True, ) if not response.ok: - logger.error(f" ** Error received: {response}") err = response.json()["error"] - logger.error(f' {err["name"]}, {err["statusCode"]}: {err["message"]}') + logger.error( + f'Error retrieving token for user: {err["name"]}, {err["statusCode"]}: {err["message"]}' + ) raise ScicatLoginError(response.content) data = response.json() diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index baa9053..c9069fc 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -1,6 +1,7 @@ from datetime import datetime from pathlib import Path +import pytest import requests_mock from ..client import ( from_credentials, @@ -8,6 +9,7 @@ encode_thumbnail, get_file_mod_time, get_file_size, + ScicatCommError, ) from ..model import ( @@ -111,6 +113,71 @@ def test_scicat_ingest(): scicat.upload_attachment(attachment) +def test_get_dataset(): + with requests_mock.Mocker() as mock_request: + dataset = RawDataset( + size=42, + owner="slartibartfast", + contactEmail="slartibartfast@magrathea.org", + creationLocation="magrathea", + creationTime=str(datetime.now()), + instrumentId="earth", + proposalId="deepthought", + dataFormat="planet", + principalInvestigator="A. Mouse", + sourceFolder="/foo/bar", + scientificMetadata={"a": "field"}, + sampleId="gargleblaster", + ownerGroup="magrathea", + accessGroups=["deep_though"], + ) + mock_request.get( + local_url + "Datasets/123", json=dataset.dict(exclude_none=True) + ) + + client = from_token(base_url=local_url, token="a_token") + retrieved = client.datasets_get_one("123") + assert retrieved == dataset.dict(exclude_none=True) + + +def test_get_nonexistent_dataset(): + with requests_mock.Mocker() as mock_request: + mock_request.get( + local_url + "Datasets/74", + status_code=404, + reason="Not Found", + json={ + "error": { + "statusCode": 404, + "name": "Error", + "message": 'Unknown "Dataset" id "74".', + "code": "MODEL_NOT_FOUND", + } + }, + ) + client = from_token(base_url=local_url, token="a_token") + assert client.datasets_get_one("74") is None + + +def test_get_dataset_bad_url(): + with requests_mock.Mocker() as mock_request: + mock_request.get( + "http://localhost:3000/api/v100/Datasets/53", + status_code=404, + reason="Not Found", + json={ + "error": { + "statusCode": 404, + "name": "Error", + "message": "Cannot GET /api/v100/Datasets/53", + } + }, + ) + client = from_token(base_url="http://localhost:3000/api/v100", token="a_token") + with pytest.raises(ScicatCommError): + client.datasets_get_one("53") + + def test_initializers(): with requests_mock.Mocker() as mock_request: add_mock_requests(mock_request) diff --git a/pyscicat/tests/test_suite_2.py b/pyscicat/tests/test_suite_2.py index 5575b2c..8384dfa 100644 --- a/pyscicat/tests/test_suite_2.py +++ b/pyscicat/tests/test_suite_2.py @@ -111,14 +111,14 @@ def test_scicat_ingest_raw_dataset(): # Create Dataset dataset = RawDataset(**data["dataset"], **ownable.dict()) - created_dataset = scicat.create_dataset(dataset) + created_dataset_pid = scicat.create_dataset(dataset) - assert created_dataset["pid"] == data["id"] + assert created_dataset_pid == data["id"] # origDatablock with DataFiles origDataBlock = OrigDatablock( size=data["orig_datablock"]["size"], - datasetId=created_dataset["pid"], + datasetId=created_dataset_pid, dataFileList=[ DataFile(**file) for file in data["orig_datablock"]["dataFileList"] ], @@ -147,14 +147,14 @@ def test_scicat_ingest_derived_dataset(): # Create Dataset dataset = RawDataset(**data["dataset"], **ownable.dict()) - created_dataset = scicat.create_dataset(dataset) + created_dataset_pid = scicat.create_dataset(dataset) - assert created_dataset["pid"] == data["id"] + assert created_dataset_pid == data["id"] # origDatablock with DataFiles origDataBlock = OrigDatablock( size=data["orig_datablock"]["size"], - datasetId=created_dataset["pid"], + datasetId=created_dataset_pid, dataFileList=[ DataFile(**file) for file in data["orig_datablock"]["dataFileList"] ], From 0650844ce8d4c0ef5ec8841dee154fb8c4d0fe07 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 17 Aug 2022 16:35:42 +0200 Subject: [PATCH 54/98] Remove log call in client constructor --- pyscicat/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index b764960..6c6b3a8 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -80,8 +80,6 @@ def __init__( self._token = token # store token here self._headers = {} # store headers - logger.info(f"Starting ingestor talking to scicat at: {self._base_url}") - if not self._token: assert (self._username is not None) and ( self._password is not None From 6c0b7c096fc682c3e675b9b99389afbd1dbe3a1a Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 29 Sep 2022 16:40:34 +0200 Subject: [PATCH 55/98] Log in via either Users/login or auth/msad --- pyscicat/client.py | 57 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index d71dcf4..667d213 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -6,7 +6,7 @@ import logging import json from typing import List -import urllib +import urllib.parse import requests @@ -839,23 +839,52 @@ def from_credentials(base_url: str, username: str, password: str): return from_token(base_url, token) -def get_token(base_url, username, password): - """logs in using the provided username / password combination - and receives token for further communication use""" - logger.info(" Getting new token") - if base_url[-1] != "/": - base_url = base_url + "/" +def _log_in_via_users_login(base_url, username, password): + response = requests.post( + urllib.parse.urljoin(base_url, "Users/login"), + json={"username": username, "password": password}, + stream=False, + verify=True, + ) + if not response.ok: + logger.info(f" Failed to log in via endpoint Users/login: {response.json()}") + return response + + +def _log_in_via_auth_msad(base_url, username, password): + import re + + # Strip the api/vn suffix + base_url = re.sub(r"/api/v\d+/?", "", base_url) response = requests.post( - base_url + "Users/login", + urllib.parse.urljoin(base_url, "auth/msad"), json={"username": username, "password": password}, stream=False, verify=True, ) if not response.ok: - logger.error(f" ** Error received: {response}") - err = response.json()["error"] - logger.error(f' {err["name"]}, {err["statusCode"]}: {err["message"]}') - raise ScicatLoginError(response.content) + logger.info(f" Failed to log in via auth/msad: {response.json()}") + return response + - data = response.json() - return data["id"] # not sure if semantically correct +def get_token(base_url, username, password): + """logs in using the provided username / password combination + and receives token for further communication use""" + # Users/login only works for functional accounts and auth/msad for regular users. + # Try both and see what works. This is not nice but seems to be the only + # feasible solution right now. + logger.info(" Getting new token") + + response = _log_in_via_users_login(base_url, username, password) + if response.ok: + return response.json()["id"] # not sure if semantically correct + + response = _log_in_via_auth_msad(base_url, username, password) + if response.ok: + return response.json()["access_token"] + + err = response.json()["error"] + logger.error( + f' Failed log in: {err["name"]}, {err["statusCode"]}: {err["message"]}' + ) + raise ScicatLoginError(response.content) From 07c1b5f6a8f6773e914b0e9e5761e36aa880c483 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 9 Nov 2022 08:59:59 +0100 Subject: [PATCH 56/98] Fix call to urljoin --- pyscicat/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index abe7d60..0c4fc46 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -731,7 +731,7 @@ def from_credentials(base_url: str, username: str, password: str): def _log_in_via_users_login(base_url, username, password): response = requests.post( - urllib.parse.urljoin(base_url, "Users/login"), + urljoin(base_url, "Users/login"), json={"username": username, "password": password}, stream=False, verify=True, @@ -747,7 +747,7 @@ def _log_in_via_auth_msad(base_url, username, password): # Strip the api/vn suffix base_url = re.sub(r"/api/v\d+/?", "", base_url) response = requests.post( - urllib.parse.urljoin(base_url, "auth/msad"), + urljoin(base_url, "auth/msad"), json={"username": username, "password": password}, stream=False, verify=True, From 716b0146d1f7ad87834b40dfe9a6c2bd66bcd4d6 Mon Sep 17 00:00:00 2001 From: Brian Richard Pauw Date: Thu, 10 Nov 2022 19:00:12 +0100 Subject: [PATCH 57/98] small fixes to model and search --- pyscicat/client.py | 4 ++++ pyscicat/model.py | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 667d213..b0e829c 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -542,6 +542,10 @@ def datasets_get_many(self, filter_fields=None) -> List[Dataset]: ```python filterField = {"proposalId": ""} ``` + If you want to search on partial strings, you can use "like": + ```python + filterField = {"proposalId": {"like":"123"}} + ``` Parameters ---------- diff --git a/pyscicat/model.py b/pyscicat/model.py index 2cfaf13..bcaab4b 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -135,6 +135,7 @@ class Dataset(Ownable): type: DatasetType validationStatus: Optional[str] version: Optional[str] + scientificMetadata: Optional[Dict] class RawDataset(Dataset): @@ -144,13 +145,11 @@ class RawDataset(Dataset): principalInvestigator: Optional[str] creationLocation: Optional[str] - dataFormat: str type: DatasetType = DatasetType.raw dataFormat: Optional[str] endTime: Optional[str] # datetime sampleId: Optional[str] proposalId: Optional[str] - scientificMetadata: Optional[Dict] class DerivedDataset(Dataset): @@ -163,7 +162,6 @@ class DerivedDataset(Dataset): usedSoftware: List[str] jobParameters: Optional[dict] jobLogData: Optional[str] - scientificMetadata: Optional[Dict] type: DatasetType = DatasetType.derived From b220051c6430b2eca1e2d0355d81670f3605fd8a Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Fri, 27 Jan 2023 12:27:57 +0100 Subject: [PATCH 58/98] Implement upload functions for proposals, samples and instruments --- pyscicat/client.py | 97 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index d0b8a0f..4d9fbc7 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -16,9 +16,12 @@ Attachment, Datablock, Dataset, + DerivedDataset, + Instrument, OrigDatablock, + Proposal, RawDataset, - DerivedDataset, + Sample, ) logger = logging.getLogger("splash_ingest") @@ -413,6 +416,98 @@ def datasets_attachment_create( upload_attachment = datasets_attachment_create create_dataset_attachment = datasets_attachment_create + def samples_create(self, sample: Sample) -> str: + """ + Create a new sample or update an existing one. + This function is also accessible as upload_sample. + + + Parameters + ---------- + sample : Sample + Sample to upload + + Returns + ------- + str + id of the newly created sample + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + return self._call_endpoint( + cmd="post", + endpoint="Samples", + data=sample, + operation="samples_create", + ).get("sampleId") + + upload_sample = samples_create + + def instruments_create(self, instrument: Instrument): + """ + Create a new instrument or update an existing one. + Note that in SciCat admin rights are required to upload instruments. + This function is also accessible as upload_instrument. + + + Parameters + ---------- + instrument : Instrument + Instrument to upload + + Returns + ------- + str + pid (or unique identifier) of the newly created instrument + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + return self._call_endpoint( + cmd="post", + endpoint="Instruments", + data=instrument, + operation="instruments_create", + ).get("pid") + + upload_instrument = instruments_create + + def proposals_create(self, proposal: Proposal): + """ + Create a new proposal or update an existing one. + Note that in SciCat admin rights are required to upload proposals. + This function is also accessible as upload_proposal. + + + Parameters + ---------- + proposal : Proposal + Proposal to upload + + Returns + ------- + str + id of the newly created proposal + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + return self._call_endpoint( + cmd="post", + endpoint="Proposals", + data=proposal, + operation="proposals_create", + ).get("proposalId") + + upload_proposal = proposals_create + def datasets_find( self, skip: int = 0, limit: int = 25, query_fields: Optional[dict] = None ) -> Optional[dict]: From 3f8cd33a6032edd58ae9311d55df07d05d0aba6a Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Fri, 27 Jan 2023 12:58:06 +0100 Subject: [PATCH 59/98] Test upload functions for proposals, samples and instruments --- pyscicat/tests/test_client.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index c9069fc..685033b 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -16,7 +16,10 @@ Attachment, Datablock, DataFile, + Instrument, + Proposal, RawDataset, + Sample, Ownable, ) @@ -28,7 +31,11 @@ def add_mock_requests(mock_request): local_url + "Users/login", json={"id": "a_token"}, ) - mock_request.post(local_url + "Samples", json={"sampleId": "dataset_id"}) + + mock_request.post(local_url + "Instruments", json={"pid": "earth"}) + mock_request.post(local_url + "Proposals", json={"proposalId": "deepthought"}) + mock_request.post(local_url + "Samples", json={"sampleId": "gargleblaster"}) + mock_request.post(local_url + "RawDatasets/replaceOrCreate", json={"pid": "42"}) mock_request.patch( local_url + "Datasets/42", @@ -66,6 +73,30 @@ def test_scicat_ingest(): size = get_file_size(thumb_path) assert size is not None + # Instrument + instrument = Instrument( + pid="earth", name="Earth", customMetadata={"a": "field"} + ) + assert scicat.upload_instrument(instrument) == "earth" + assert scicat.instruments_create(instrument) == "earth" + + # Proposal + proposal = Proposal( + proposalId="deepthought", title="Deepthought", **ownable.dict() + ) + assert scicat.upload_proposal(proposal) == "deepthought" + assert scicat.proposals_create(proposal) == "deepthought" + + # Sample + sample = Sample( + sampleId="gargleblaster", + description="Gargleblaster", + sampleCharacteristics={"a": "field"}, + **ownable.dict() + ) + assert scicat.upload_sample(sample) == "gargleblaster" + assert scicat.samples_create(proposal) == "gargleblaster" + # RawDataset dataset = RawDataset( path="/foo/bar", From 6216e100360529f1e6ce7975222d0afaee2c282d Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Fri, 27 Jan 2023 14:39:50 +0100 Subject: [PATCH 60/98] Update data model --- pyscicat/model.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pyscicat/model.py b/pyscicat/model.py index bcaab4b..837e9bf 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -26,7 +26,8 @@ class Ownable(MongoQueryable): """Many objects in SciCat are ownable""" ownerGroup: str - accessGroups: List[str] + accessGroups: Optional[List[str]] + instrumentGroup: Optional[str] class User(BaseModel): @@ -45,15 +46,14 @@ class Proposal(Ownable): Defines the purpose of an experiment and links an experiment to principal investigator and main proposer """ - # TODO: find out which of these are not optional and update - proposalId: Optional[str] + proposalId: str pi_email: Optional[str] pi_firstname: Optional[str] pi_lastname: Optional[str] - email: Optional[str] + email: str firstname: Optional[str] lastname: Optional[str] - title: Optional[str] + title: Optional[str] # required in next backend version abstract: Optional[str] startTime: Optional[str] endTime: Optional[str] @@ -68,7 +68,6 @@ class Sample(Ownable): Raw datasets should be linked to such sample definitions. """ - # TODO: find out which of these are not optional and update sampleId: Optional[str] owner: Optional[str] description: Optional[str] From 15364eff9a898e11ae9f9a5b3715c40e6ec31fd3 Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Fri, 27 Jan 2023 16:28:23 +0100 Subject: [PATCH 61/98] Update documentation --- docs/source/howto/ingest.md | 14 ++++++++++++++ pyscicat/client.py | 3 +++ 2 files changed, 17 insertions(+) diff --git a/docs/source/howto/ingest.md b/docs/source/howto/ingest.md index 05cf37d..a694ed0 100644 --- a/docs/source/howto/ingest.md +++ b/docs/source/howto/ingest.md @@ -13,6 +13,7 @@ from pyscicat.model import ( Datablock, DataFile, Dataset, + Sample, Ownable ) @@ -61,6 +62,19 @@ Note that we store the provided dataset_id in a variable for later use. Also note the `sourceFolder`. This is a folder on the file system that SciCat has access to, and will contain the files for this `Dataset`. +Proposals and instruments have to be created by an administrator. A sample with `sampleId="gargleblaster"` can be created like this: +```python +sample = Sample( + sampleId="gargleblaster", + owner="Chamber of Commerce", + description="A legendary drink.", + sampleCharacteristics={"Flavour": "Unknown, but potent"}, + isPublished=False, + **ownable.dict() +) +sample_id = client.upload_sample(sample) # sample_id == "gargleblaster" +``` + ## Upload a Datablock ```python diff --git a/pyscicat/client.py b/pyscicat/client.py index 4d9fbc7..14bcdf8 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -419,6 +419,7 @@ def datasets_attachment_create( def samples_create(self, sample: Sample) -> str: """ Create a new sample or update an existing one. + An error is raised when a sample with the same sampleId already exists. This function is also accessible as upload_sample. @@ -450,6 +451,7 @@ def instruments_create(self, instrument: Instrument): """ Create a new instrument or update an existing one. Note that in SciCat admin rights are required to upload instruments. + An error is raised when an instrument with the same pid already exists. This function is also accessible as upload_instrument. @@ -481,6 +483,7 @@ def proposals_create(self, proposal: Proposal): """ Create a new proposal or update an existing one. Note that in SciCat admin rights are required to upload proposals. + An error is raised when a proposal with the same proposalId already exists. This function is also accessible as upload_proposal. From bfed3c88c61e310655fbae041eef36e34dfb1991 Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Fri, 27 Jan 2023 17:30:00 +0100 Subject: [PATCH 62/98] Implement update functions for proposals, samples and instruments --- pyscicat/client.py | 100 +++++++++++++++++++++++++++++++++- pyscicat/tests/test_client.py | 17 +++++- 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 14bcdf8..7bb7c70 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -431,7 +431,7 @@ def samples_create(self, sample: Sample) -> str: Returns ------- str - id of the newly created sample + ID of the newly created sample Raises ------ @@ -447,6 +447,38 @@ def samples_create(self, sample: Sample) -> str: upload_sample = samples_create + def update_sample(self, sample: Sample, sampleId: str = None) -> str: + """Updates an existing sample + + Parameters + ---------- + sample : Sample + Sample to update + + sampleId + ID of sample being updated. By default, ID is taken from sample parameter. + + Returns + ------- + str + ID of the sample + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + if sampleId is None: + assert sample.sampleId is not None, "sampleId should not be None" + sampleId = sample.sampleId + sample.sampleId = None + return self._call_endpoint( + cmd="patch", + endpoint=f"Samples/{quote_plus(sampleId)}", + data=sample, + operation="update_sample", + ).get("sampleId") + def instruments_create(self, instrument: Instrument): """ Create a new instrument or update an existing one. @@ -479,6 +511,39 @@ def instruments_create(self, instrument: Instrument): upload_instrument = instruments_create + def update_instrument(self, instrument: Instrument, pid: str = None) -> str: + """Updates an existing instrument. + Note that in SciCat admin rights are required to upload instruments. + + Parameters + ---------- + instrument : Instrument + Instrument to update + + pid + pid (or unique identifier) of instrument being updated. By default, pid is taken from instrument parameter. + + Returns + ------- + str + ID of the instrument + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + if pid is None: + assert instrument.pid is not None, "pid should not be None" + pid = instrument.pid + instrument.pid = None + return self._call_endpoint( + cmd="patch", + endpoint=f"Instruments/{quote_plus(pid)}", + data=instrument, + operation="update_instrument", + ).get("pid") + def proposals_create(self, proposal: Proposal): """ Create a new proposal or update an existing one. @@ -511,6 +576,39 @@ def proposals_create(self, proposal: Proposal): upload_proposal = proposals_create + def update_proposal(self, proposal: Proposal, proposalId: str = None) -> str: + """Updates an existing proposal. + Note that in SciCat admin rights are required to upload proposals. + + Parameters + ---------- + proposal : Proposal + Proposal to update + + proposalId + ID of proposal being updated. By default, this is taken from proposal parameter. + + Returns + ------- + str + ID of the proposal + + Raises + ------ + ScicatCommError + Raises if a non-20x message is returned + """ + if proposalId is None: + assert proposal.proposalId is not None, "proposalId should not be None" + proposalId = proposal.proposalId + proposal.proposalId = None + return self._call_endpoint( + cmd="patch", + endpoint=f"Proposals/{quote_plus(proposalId)}", + data=proposal, + operation="update_proposal", + ).get("proposalId") + def datasets_find( self, skip: int = 0, limit: int = 25, query_fields: Optional[dict] = None ) -> Optional[dict]: diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 685033b..3bc535f 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -35,6 +35,13 @@ def add_mock_requests(mock_request): mock_request.post(local_url + "Instruments", json={"pid": "earth"}) mock_request.post(local_url + "Proposals", json={"proposalId": "deepthought"}) mock_request.post(local_url + "Samples", json={"sampleId": "gargleblaster"}) + mock_request.patch(local_url + "Instruments/earth", json={"pid": "earth"}) + mock_request.patch( + local_url + "Proposals/deepthought", json={"proposalId": "deepthought"} + ) + mock_request.patch( + local_url + "Samples/gargleblaster", json={"sampleId": "gargleblaster"} + ) mock_request.post(local_url + "RawDatasets/replaceOrCreate", json={"pid": "42"}) mock_request.patch( @@ -79,13 +86,18 @@ def test_scicat_ingest(): ) assert scicat.upload_instrument(instrument) == "earth" assert scicat.instruments_create(instrument) == "earth" + assert scicat.update_instrument(instrument) == "earth" # Proposal proposal = Proposal( - proposalId="deepthought", title="Deepthought", **ownable.dict() + proposalId="deepthought", + title="Deepthought", + email="deepthought@viltvodle.com", + **ownable.dict() ) assert scicat.upload_proposal(proposal) == "deepthought" assert scicat.proposals_create(proposal) == "deepthought" + assert scicat.update_proposal(proposal) == "deepthought" # Sample sample = Sample( @@ -95,7 +107,8 @@ def test_scicat_ingest(): **ownable.dict() ) assert scicat.upload_sample(sample) == "gargleblaster" - assert scicat.samples_create(proposal) == "gargleblaster" + assert scicat.samples_create(sample) == "gargleblaster" + assert scicat.update_sample(sample) == "gargleblaster" # RawDataset dataset = RawDataset( From a3a4276731e9d9e6259d2b1dd6368bf8525fc7a3 Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Fri, 27 Jan 2023 18:16:59 +0100 Subject: [PATCH 63/98] Fix documentation --- pyscicat/client.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 7bb7c70..b28a743 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -418,7 +418,7 @@ def datasets_attachment_create( def samples_create(self, sample: Sample) -> str: """ - Create a new sample or update an existing one. + Create a new sample. An error is raised when a sample with the same sampleId already exists. This function is also accessible as upload_sample. @@ -467,6 +467,9 @@ def update_sample(self, sample: Sample, sampleId: str = None) -> str: ------ ScicatCommError Raises if a non-20x message is returned + + AssertionError + Raises if no ID is provided """ if sampleId is None: assert sample.sampleId is not None, "sampleId should not be None" @@ -481,7 +484,7 @@ def update_sample(self, sample: Sample, sampleId: str = None) -> str: def instruments_create(self, instrument: Instrument): """ - Create a new instrument or update an existing one. + Create a new instrument. Note that in SciCat admin rights are required to upload instruments. An error is raised when an instrument with the same pid already exists. This function is also accessible as upload_instrument. @@ -532,6 +535,9 @@ def update_instrument(self, instrument: Instrument, pid: str = None) -> str: ------ ScicatCommError Raises if a non-20x message is returned + + AssertionError + Raises if no ID is provided """ if pid is None: assert instrument.pid is not None, "pid should not be None" @@ -546,7 +552,7 @@ def update_instrument(self, instrument: Instrument, pid: str = None) -> str: def proposals_create(self, proposal: Proposal): """ - Create a new proposal or update an existing one. + Create a new proposal. Note that in SciCat admin rights are required to upload proposals. An error is raised when a proposal with the same proposalId already exists. This function is also accessible as upload_proposal. @@ -560,7 +566,7 @@ def proposals_create(self, proposal: Proposal): Returns ------- str - id of the newly created proposal + ID of the newly created proposal Raises ------ @@ -597,6 +603,9 @@ def update_proposal(self, proposal: Proposal, proposalId: str = None) -> str: ------ ScicatCommError Raises if a non-20x message is returned + + AssertionError + Raises if no ID is provided """ if proposalId is None: assert proposal.proposalId is not None, "proposalId should not be None" From dc86102487e55ee89266dcdb058daf7861dff576 Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Fri, 27 Jan 2023 19:31:56 +0100 Subject: [PATCH 64/98] Flake8 compliance --- pyscicat/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index b28a743..37adaaf 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -524,7 +524,8 @@ def update_instrument(self, instrument: Instrument, pid: str = None) -> str: Instrument to update pid - pid (or unique identifier) of instrument being updated. By default, pid is taken from instrument parameter. + pid (or unique identifier) of instrument being updated. + By default, pid is taken from instrument parameter. Returns ------- From 1ee0bd3e64d41ef436b3ec62be9c0a11804b8c70 Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Mon, 30 Jan 2023 11:07:12 +0100 Subject: [PATCH 65/98] Update-function of the form _update --- pyscicat/client.py | 12 ++++++------ pyscicat/tests/test_client.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 37adaaf..d381e1f 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -447,7 +447,7 @@ def samples_create(self, sample: Sample) -> str: upload_sample = samples_create - def update_sample(self, sample: Sample, sampleId: str = None) -> str: + def samples_update(self, sample: Sample, sampleId: str = None) -> str: """Updates an existing sample Parameters @@ -479,7 +479,7 @@ def update_sample(self, sample: Sample, sampleId: str = None) -> str: cmd="patch", endpoint=f"Samples/{quote_plus(sampleId)}", data=sample, - operation="update_sample", + operation="samples_update", ).get("sampleId") def instruments_create(self, instrument: Instrument): @@ -514,7 +514,7 @@ def instruments_create(self, instrument: Instrument): upload_instrument = instruments_create - def update_instrument(self, instrument: Instrument, pid: str = None) -> str: + def instruments_update(self, instrument: Instrument, pid: str = None) -> str: """Updates an existing instrument. Note that in SciCat admin rights are required to upload instruments. @@ -548,7 +548,7 @@ def update_instrument(self, instrument: Instrument, pid: str = None) -> str: cmd="patch", endpoint=f"Instruments/{quote_plus(pid)}", data=instrument, - operation="update_instrument", + operation="instruments_update", ).get("pid") def proposals_create(self, proposal: Proposal): @@ -583,7 +583,7 @@ def proposals_create(self, proposal: Proposal): upload_proposal = proposals_create - def update_proposal(self, proposal: Proposal, proposalId: str = None) -> str: + def proposals_update(self, proposal: Proposal, proposalId: str = None) -> str: """Updates an existing proposal. Note that in SciCat admin rights are required to upload proposals. @@ -616,7 +616,7 @@ def update_proposal(self, proposal: Proposal, proposalId: str = None) -> str: cmd="patch", endpoint=f"Proposals/{quote_plus(proposalId)}", data=proposal, - operation="update_proposal", + operation="proposals_update", ).get("proposalId") def datasets_find( diff --git a/pyscicat/tests/test_client.py b/pyscicat/tests/test_client.py index 3bc535f..6d036f0 100644 --- a/pyscicat/tests/test_client.py +++ b/pyscicat/tests/test_client.py @@ -86,7 +86,7 @@ def test_scicat_ingest(): ) assert scicat.upload_instrument(instrument) == "earth" assert scicat.instruments_create(instrument) == "earth" - assert scicat.update_instrument(instrument) == "earth" + assert scicat.instruments_update(instrument) == "earth" # Proposal proposal = Proposal( @@ -97,7 +97,7 @@ def test_scicat_ingest(): ) assert scicat.upload_proposal(proposal) == "deepthought" assert scicat.proposals_create(proposal) == "deepthought" - assert scicat.update_proposal(proposal) == "deepthought" + assert scicat.proposals_update(proposal) == "deepthought" # Sample sample = Sample( @@ -108,7 +108,7 @@ def test_scicat_ingest(): ) assert scicat.upload_sample(sample) == "gargleblaster" assert scicat.samples_create(sample) == "gargleblaster" - assert scicat.update_sample(sample) == "gargleblaster" + assert scicat.samples_update(sample) == "gargleblaster" # RawDataset dataset = RawDataset( From 336fc874ec7c7c618133959ddc0d7ad58047970c Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Mon, 30 Jan 2023 11:34:09 +0100 Subject: [PATCH 66/98] Rename update_dataset() to datasets_update() in client. --- pyscicat/client.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index d381e1f..9d3e05a 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -211,7 +211,7 @@ def datasets_raw_replace(self, dataset: Dataset) -> str: This function was renamed. It is still accessible with the original name for backward compatibility The original names were repalce_raw_dataset and upload_raw_dataset - THis function is obsolete and it will be removed in future releases + This function is obsolete and it will be removed in future releases Parameters ---------- @@ -272,8 +272,11 @@ def datasets_derived_replace(self, dataset: Dataset) -> str: operation="datasets_derived_replace", ).get("pid") - def update_dataset(self, dataset: Dataset, pid: str) -> str: + def datasets_update(self, dataset: Dataset, pid: str) -> str: """Updates an existing dataset + This function was renamed. + It is still accessible with the original name for backward compatibility + The original name was update_dataset. Parameters ---------- @@ -296,9 +299,15 @@ def update_dataset(self, dataset: Dataset, pid: str) -> str: cmd="patch", endpoint=f"Datasets/{quote_plus(pid)}", data=dataset, - operation="update_dataset", + operation="datasets_update", ).get("pid") + """ + Update a dataset + Original name, kept for for backward compatibility + """ + update_dataset = datasets_update + def datasets_datablock_create( self, datablock: Datablock, datasetType: str = "RawDatasets" ) -> dict: From ea0e048639d19eaef9d3721736f42fbffb5efb7e Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Thu, 16 Mar 2023 13:44:20 +0000 Subject: [PATCH 67/98] adding some tests to work against a scicat v4 backend. --- pyscicat/tests/tests_local.py | 78 +++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 pyscicat/tests/tests_local.py diff --git a/pyscicat/tests/tests_local.py b/pyscicat/tests/tests_local.py new file mode 100644 index 0000000..bef285b --- /dev/null +++ b/pyscicat/tests/tests_local.py @@ -0,0 +1,78 @@ +import unittest +from ..client import ScicatClient +from ..model import RawDataset +from datetime import datetime +import os + +class TestClientLocally(unittest.TestCase): + """ + These tests do not use mocks and are designed to connect to a v4 service for Scicat backend. You can run this easily + in docker-compose following the repo https://github.com/SciCatProject/scicatlive. You will also need to use one of + the default user accounts or add your own. + + You will need to set environmental variables for + BASE_URL - the url of your scicat service e.g. http://localhost:3000/api/v3 + SCICAT_USER - the name of your scicat user. + SCICAT_PASSWORD - the password for your scicat user. + """ + + def test_client(self): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password="SCICAT_PASSWORD") + self.assertIsInstance(sci_clie, ScicatClient) + print(sci_clie._token) + + + + def test_upload_dataset(self): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password="SCICAT_PASSWORD") + + payload = RawDataset( + datasetName="a guide book", + path="/foo/bar", + size=42, + owner=os.environ["SCICAT_USER"], + ownerGroup="Magrateheans", + contactEmail="slartibartfast@magrathea.org", + creationLocation="magrathea", + creationTime= datetime.isoformat(datetime.now()), + instrumentId="earth", + proposalId="deepthought", + dataFormat="planet", + principalInvestigator="A. Mouse", + sourceFolder="/foo/bar", + scientificMetadata={"a": "field"}, + sampleId="gargleblaster", + accessGroups=[] + ) + + sci_clie.upload_new_dataset(payload) + + + def test_replace_dataset(self): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password="SCICAT_PASSWORD") + + pid="PID.SAMPLE.PREFIX48a8f164-166a-4557-bafc-5a7362e39fe7" + payload= RawDataset( + size=142, + owner="slartibartfast", + ownerGroup="Magrateheans", + contactEmail="slartibartfast@magrathea.org", + creationLocation="magrathea", + creationTime=datetime.isoformat(datetime.now()), + instrumentId="earth", + proposalId="deepthought", + dataFormat="planet", + principalInvestigator="A. Mouse", + sourceFolder="/foo/bar", + scientificMetadata={"a": "field"}, + sampleId="gargleblaster", + accessGroups=["Vogons"] + ) + sci_clie.update_dataset(payload, pid) + + + def test_get_dataset(self): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password="SCICAT_PASSWORD") + datasets = sci_clie.get_datasets({"owner":os.environ["SCICAT_USER"]}) + with self.subTest(dataset=datasets): + self.assertEqual(dataset["owner"], os.environ["SCICAT_USER"]) From 7c76635a47746bf0eb53448c7b31b7f562876263 Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Fri, 17 Mar 2023 13:57:17 +0000 Subject: [PATCH 68/98] Restructured the code so that we can install the optional requiments using `pip install -e ".[option]"`. To add optional requirements add to the [optional.extras_require] in the setup config. All functions and imports work the same way. Restructured the testing suite so it follows a similar pattern. All tests pass. --- pyscicat/__init__.py | 2 + setup.cfg | 51 ++++++++++++++ setup.py | 64 +----------------- .../cylinderHex_r5_s12_T50_large_ranW_0p5.nxs | Bin .../test_hdf5}/test_hdf5sct.py | 8 +-- .../tests => tests/test_pyscicat}/__init__.py | 0 .../tests => tests/test_pyscicat}/conftest.py | 0 .../test_pyscicat}/data/SciCatLogo.png | Bin .../test_pyscicat}/test_client.py | 4 +- 9 files changed, 60 insertions(+), 69 deletions(-) rename {pyscicat/hdf5/_tests/testdata => tests/test_hdf5/data}/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs (100%) rename {pyscicat/hdf5/_tests => tests/test_hdf5}/test_hdf5sct.py (83%) rename {pyscicat/tests => tests/test_pyscicat}/__init__.py (100%) rename {pyscicat/tests => tests/test_pyscicat}/conftest.py (100%) rename {pyscicat/tests => tests/test_pyscicat}/data/SciCatLogo.png (100%) rename {pyscicat/tests => tests/test_pyscicat}/test_client.py (99%) diff --git a/pyscicat/__init__.py b/pyscicat/__init__.py index 80edaf0..4f9eb34 100644 --- a/pyscicat/__init__.py +++ b/pyscicat/__init__.py @@ -2,3 +2,5 @@ __version__ = get_versions()["version"] del get_versions +from . import hdf5 +from . import ingest \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index c80108f..4a3f8f1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,3 +4,54 @@ style = pep440-post versionfile_source = pyscicat/_version.py versionfile_build = pyscicat/_version.py tag_prefix = v + +[metadata] + +name = pyscicat +version = 1.0.0 +description = a python API to communicate with the Scicat API +long_description = file: README.md +long_description_content_type = text/markdown +author = Dylan McReynolds +author_email = "dmcreynolds@lbl.gov", +url = "https://github.com/scicatproject/pyscicat" +license_file = LICENSE +license="BSD (3-clause)" +classifiers= + "Development Status :: 2 - Pre-Alpha" + "Natural Language :: English" + "Programming Language :: Python :: 3" + + +[options] +include_package_data = True +install_requires = + pydantic + requests +packages = find: +python_requires = >=3.8 + + +[options.extras_require] +hdf5 = + hdf5plugin + h5py +dev = + codecov + coverage + flake8 + pytest + sphinx + twine + black + requests_mock +docs = + ipython + matplotlib + mistune <2.0.0 # temporary while sphinx sorts this out + myst-parser + numpydoc + sphinx-click + sphinx-copybutton + sphinxcontrib.openapi + sphinx_rtd_theme \ No newline at end of file diff --git a/setup.py b/setup.py index 9e430c0..9709c4e 100644 --- a/setup.py +++ b/setup.py @@ -1,68 +1,6 @@ from pathlib import Path from setuptools import setup, find_packages import sys -import versioneer -min_version = (3, 7) -if sys.version_info < min_version: - error = """ -pyscicat does not support Python {0}.{1}. -Python {2}.{3} and above is required. Check your Python version like so: +setup() -python3 --version - -This may be due to an out-of-date pip. Make sure you have pip >= 9.0.1. -Upgrade pip like so: - -pip install --upgrade pip -""".format( - *(sys.version_info[:2] + min_version) - ) - sys.exit(error) - -here = Path(__file__).absolute() - -with open(here.with_name("README.md"), encoding="utf-8") as readme_file: - readme = readme_file.read() - - -def read_requirements_from_here(here: Path, filename: str = None) -> list: - assert filename is not None, "filename as string must be provided" - assert here.with_name( - filename - ).exists(), f"requirements filename {filename} does not exist" - with open(here.with_name(filename)) as requirements_file: - # Parse requirements.txt, ignoring any commented-out lines. - requirements = [ - line - for line in requirements_file.read().splitlines() - if not line.startswith("#") - ] - return requirements - - -extras_require = {} -extras_require["base"] = read_requirements_from_here(here, "requirements.txt") -extras_require["h5tools"] = read_requirements_from_here(here, "requirements-hdf5.txt") - -setup( - name="pyscicat", - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - description="Code for communicating to a SciCat backend server python", - long_description=readme, - author="Dylan McReynolds", - author_email="dmcreynolds@lbl.gov", - url="https://github.com/scicatproject/pyscicat", - python_requires=">={}".format(".".join(str(n) for n in min_version)), - packages=find_packages(exclude=["docs", "tests"]), - include_package_data=True, - extras_require=extras_require, - install_requires=extras_require["base"], - license="BSD (3-clause)", - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Natural Language :: English", - "Programming Language :: Python :: 3", - ], -) diff --git a/pyscicat/hdf5/_tests/testdata/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs b/tests/test_hdf5/data/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs similarity index 100% rename from pyscicat/hdf5/_tests/testdata/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs rename to tests/test_hdf5/data/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs diff --git a/pyscicat/hdf5/_tests/test_hdf5sct.py b/tests/test_hdf5/test_hdf5sct.py similarity index 83% rename from pyscicat/hdf5/_tests/test_hdf5sct.py rename to tests/test_hdf5/test_hdf5sct.py index 466e190..3f3af64 100644 --- a/pyscicat/hdf5/_tests/test_hdf5sct.py +++ b/tests/test_hdf5/test_hdf5sct.py @@ -9,19 +9,19 @@ def test_readValue(): # more intelligent path finding: - p = sorted(Path(".").glob("**/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs"))[0] + p = sorted(Path("").glob("**/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs"))[0] v = h5Get(p, "/sasentry1/sasdata1/I") assert v != "none", "Did not extract value" def test_readAttribute(): - p = sorted(Path(".").glob("**/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs"))[0] + p = sorted(Path("").glob("**/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs"))[0] v = h5Get(p, "/sasentry1/sasdata1@timestamp") assert v != "none", "Did not extract attribute" def test_readMixedDict(): - p = sorted(Path(".").glob("**/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs"))[0] + p = sorted(Path("").glob("**/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs"))[0] v = h5GetDict( p, { @@ -38,7 +38,7 @@ def test_readMixedDict(): def test_readMetadata_withroot(): - p = sorted(Path(".").glob("**/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs"))[0] + p = sorted(Path("").glob("**/cylinderHex_r5_s12_T50_large_ranW_0p5.nxs"))[0] assert p.exists(), f"HDF5/NeXus test file: {p.as_posix()} cannot be found" resultDict = scientific_metadata(p, excludeRootEntry=True, skipKeyList=["sasdata1"]) assert resultDict is not None, "scientific_metadata has not returned anything" diff --git a/pyscicat/tests/__init__.py b/tests/test_pyscicat/__init__.py similarity index 100% rename from pyscicat/tests/__init__.py rename to tests/test_pyscicat/__init__.py diff --git a/pyscicat/tests/conftest.py b/tests/test_pyscicat/conftest.py similarity index 100% rename from pyscicat/tests/conftest.py rename to tests/test_pyscicat/conftest.py diff --git a/pyscicat/tests/data/SciCatLogo.png b/tests/test_pyscicat/data/SciCatLogo.png similarity index 100% rename from pyscicat/tests/data/SciCatLogo.png rename to tests/test_pyscicat/data/SciCatLogo.png diff --git a/pyscicat/tests/test_client.py b/tests/test_pyscicat/test_client.py similarity index 99% rename from pyscicat/tests/test_client.py rename to tests/test_pyscicat/test_client.py index 6d036f0..c165007 100644 --- a/pyscicat/tests/test_client.py +++ b/tests/test_pyscicat/test_client.py @@ -3,7 +3,7 @@ import pytest import requests_mock -from ..client import ( +from pyscicat.client import ( from_credentials, from_token, encode_thumbnail, @@ -12,7 +12,7 @@ ScicatCommError, ) -from ..model import ( +from pyscicat.model import ( Attachment, Datablock, DataFile, From 76f14a198afaef6c123e2b4446e8104c6f9c3504 Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Fri, 17 Mar 2023 15:09:40 +0000 Subject: [PATCH 69/98] moving the extra tests into the new test suite --- pyscicat/tests/test_suite_2.py | 185 ------------------ .../test_pyscicat}/tests_local.py | 27 +-- 2 files changed, 15 insertions(+), 197 deletions(-) delete mode 100644 pyscicat/tests/test_suite_2.py rename {pyscicat/tests => tests/test_pyscicat}/tests_local.py (76%) diff --git a/pyscicat/tests/test_suite_2.py b/pyscicat/tests/test_suite_2.py deleted file mode 100644 index 8384dfa..0000000 --- a/pyscicat/tests/test_suite_2.py +++ /dev/null @@ -1,185 +0,0 @@ -from pathlib import Path -import urllib -import json - -import requests_mock -from ..client import ScicatClient - -from ..model import ( - DataFile, - RawDataset, - OrigDatablock, - Ownable, -) - -global test_datasets - -local_url = "http://localhost:3000/api/v3/" -test_dataset_files = { - "raw": "../../examples/data/ingestion_simulation_dataset_ess_raw_dataset.json", - "derived": "../../examples/data/ingestion_simulation_dataset_ess_derived_dataset.json", - "published_data": "../../examples/data/published_data.json", -} -test_datasets = {} - - -def set_up_test_environment(mock_request): - - global test_datasets - - # load test data - for name, path in test_dataset_files.items(): - data_file_path = Path(__file__).parent.joinpath(path).resolve() - with open(data_file_path, "r") as fh: - test_datasets[name] = json.load(fh) - - mock_request.post( - local_url + "Users/login", - json={"id": "a_token"}, - ) - - -def set_up_mock_raw_dataset(mock_request): - data = test_datasets["raw"] - - mock_request.post( - local_url + "Datasets", - json={**{"pid": data["id"]}, **data["dataset"]}, - ) - - encoded_pid = urllib.parse.quote_plus(data["id"]) - mock_request.post( - local_url + "Datasets/" + encoded_pid + "/origdatablocks", - json={ - "size": data["orig_datablock"]["size"], - "datasetId": data["id"], - "dataFileList": data["orig_datablock"]["dataFileList"], - }, - ) - - return data - - -def set_up_mock_derived_dataset(mock_request): - data = test_datasets["derived"] - - mock_request.post( - local_url + "Datasets", - json={**{"pid": data["id"]}, **data["dataset"]}, - ) - - encoded_pid = urllib.parse.quote_plus(data["id"]) - mock_request.post( - local_url + "Datasets/" + encoded_pid + "/origdatablocks", - json={ - "size": data["orig_datablock"]["size"], - "datasetId": data["id"], - "dataFileList": data["orig_datablock"]["dataFileList"], - }, - ) - - return data - - -def set_up_mock_published_data(mock_request): - data = test_datasets["published_data"] - - mock_url = local_url + "PublishedData" - print("Mock : " + mock_url) - mock_request.get( - mock_url, - json=data, - ) - - return data - - -def test_scicat_ingest_raw_dataset(): - with requests_mock.Mocker() as mock_request: - set_up_test_environment(mock_request) - data = set_up_mock_raw_dataset(mock_request) - scicat = ScicatClient( - base_url=local_url, - username="Zaphod", - password="heartofgold", - ) - assert ( - scicat._token == "a_token" - ), "scicat client set the token given by the server" - - ownable = Ownable(**data["ownable"]) - - # Create Dataset - dataset = RawDataset(**data["dataset"], **ownable.dict()) - created_dataset_pid = scicat.create_dataset(dataset) - - assert created_dataset_pid == data["id"] - - # origDatablock with DataFiles - origDataBlock = OrigDatablock( - size=data["orig_datablock"]["size"], - datasetId=created_dataset_pid, - dataFileList=[ - DataFile(**file) for file in data["orig_datablock"]["dataFileList"] - ], - **ownable.dict() - ) - created_origdatablock = scicat.create_dataset_origdatablock(origDataBlock) - assert len(created_origdatablock["dataFileList"]) == len( - data["orig_datablock"]["dataFileList"] - ) - - -def test_scicat_ingest_derived_dataset(): - with requests_mock.Mocker() as mock_request: - set_up_test_environment(mock_request) - data = set_up_mock_derived_dataset(mock_request) - scicat = ScicatClient( - base_url=local_url, - username="Zaphod", - password="heartofgold", - ) - assert ( - scicat._token == "a_token" - ), "scicat client set the token given by the server" - - ownable = Ownable(**data["ownable"]) - - # Create Dataset - dataset = RawDataset(**data["dataset"], **ownable.dict()) - created_dataset_pid = scicat.create_dataset(dataset) - - assert created_dataset_pid == data["id"] - - # origDatablock with DataFiles - origDataBlock = OrigDatablock( - size=data["orig_datablock"]["size"], - datasetId=created_dataset_pid, - dataFileList=[ - DataFile(**file) for file in data["orig_datablock"]["dataFileList"] - ], - **ownable.dict() - ) - created_origdatablock = scicat.create_dataset_origdatablock(origDataBlock) - assert len(created_origdatablock["dataFileList"]) == len( - data["orig_datablock"]["dataFileList"] - ) - - -def test_scicat_find_published_data(): - with requests_mock.Mocker() as mock_request: - set_up_test_environment(mock_request) - data = set_up_mock_published_data(mock_request) - scicat = ScicatClient( - base_url=local_url, - username="Zaphod", - password="heartofgold", - ) - assert ( - scicat._token == "a_token" - ), "scicat client set the token given by the server" - - returned_data = scicat.find_published_data() - - assert len(data) == len(returned_data) - assert data == returned_data diff --git a/pyscicat/tests/tests_local.py b/tests/test_pyscicat/tests_local.py similarity index 76% rename from pyscicat/tests/tests_local.py rename to tests/test_pyscicat/tests_local.py index bef285b..45da1f7 100644 --- a/pyscicat/tests/tests_local.py +++ b/tests/test_pyscicat/tests_local.py @@ -1,12 +1,12 @@ import unittest -from ..client import ScicatClient -from ..model import RawDataset +from pyscicat.client import ScicatClient +from pyscicat.model import RawDataset from datetime import datetime import os class TestClientLocally(unittest.TestCase): """ - These tests do not use mocks and are designed to connect to a v4 service for Scicat backend. You can run this easily + These test_pyscicat do not use mocks and are designed to connect to a v4 service for Scicat backend. You can run this easily in docker-compose following the repo https://github.com/SciCatProject/scicatlive. You will also need to use one of the default user accounts or add your own. @@ -17,14 +17,14 @@ class TestClientLocally(unittest.TestCase): """ def test_client(self): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password="SCICAT_PASSWORD") + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password=os.environ["SCICAT_PASSWORD"]) self.assertIsInstance(sci_clie, ScicatClient) print(sci_clie._token) def test_upload_dataset(self): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password="SCICAT_PASSWORD") + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password=os.environ["SCICAT_PASSWORD"]) payload = RawDataset( datasetName="a guide book", @@ -47,9 +47,16 @@ def test_upload_dataset(self): sci_clie.upload_new_dataset(payload) + def test_get_dataset(self): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password=os.environ["SCICAT_PASSWORD"]) + datasets = sci_clie.get_datasets({"ownerGroup": "Magratheans"}) + with self.subTest(dataset=datasets): + self.assertEqual(dataset["ownerGroup"], "Magratheans") + - def test_replace_dataset(self): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password="SCICAT_PASSWORD") + def test_update_dataset(self): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], + password=os.environ["SCICAT_PASSWORD"]) pid="PID.SAMPLE.PREFIX48a8f164-166a-4557-bafc-5a7362e39fe7" payload= RawDataset( @@ -71,8 +78,4 @@ def test_replace_dataset(self): sci_clie.update_dataset(payload, pid) - def test_get_dataset(self): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password="SCICAT_PASSWORD") - datasets = sci_clie.get_datasets({"owner":os.environ["SCICAT_USER"]}) - with self.subTest(dataset=datasets): - self.assertEqual(dataset["owner"], os.environ["SCICAT_USER"]) + From 29bfeb2f3e1d81e52084db9e154953f9065158cb Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Mon, 20 Mar 2023 13:44:31 +0000 Subject: [PATCH 70/98] reducing the version of python as 3.8 is not the requirment --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4a3f8f1..e8e2dad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,13 +29,13 @@ install_requires = pydantic requests packages = find: -python_requires = >=3.8 +python_requires = >=3.0 [options.extras_require] hdf5 = - hdf5plugin - h5py + hdf5plugin + h5py dev = codecov coverage From 3fc3b2e6e62c29c67b87d83a0679587234e3ac07 Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Mon, 20 Mar 2023 13:44:55 +0000 Subject: [PATCH 71/98] porting tests to pyscicat and linting with flake8 --- pyscicat/__init__.py | 4 +- setup.py | 5 +- tests/test_pyscicat/tests_local.py | 136 ++++++++++++++++------------- 3 files changed, 76 insertions(+), 69 deletions(-) diff --git a/pyscicat/__init__.py b/pyscicat/__init__.py index 4f9eb34..32ac1d7 100644 --- a/pyscicat/__init__.py +++ b/pyscicat/__init__.py @@ -1,6 +1,6 @@ +from . import hdf5 +from . import ingest from ._version import get_versions __version__ = get_versions()["version"] del get_versions -from . import hdf5 -from . import ingest \ No newline at end of file diff --git a/setup.py b/setup.py index 9709c4e..6068493 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,3 @@ -from pathlib import Path -from setuptools import setup, find_packages -import sys +from setuptools import setup setup() - diff --git a/tests/test_pyscicat/tests_local.py b/tests/test_pyscicat/tests_local.py index 45da1f7..95204bd 100644 --- a/tests/test_pyscicat/tests_local.py +++ b/tests/test_pyscicat/tests_local.py @@ -1,81 +1,91 @@ -import unittest from pyscicat.client import ScicatClient from pyscicat.model import RawDataset from datetime import datetime import os -class TestClientLocally(unittest.TestCase): - """ - These test_pyscicat do not use mocks and are designed to connect to a v4 service for Scicat backend. You can run this easily - in docker-compose following the repo https://github.com/SciCatProject/scicatlive. You will also need to use one of - the default user accounts or add your own. - You will need to set environmental variables for - BASE_URL - the url of your scicat service e.g. http://localhost:3000/api/v3 - SCICAT_USER - the name of your scicat user. - SCICAT_PASSWORD - the password for your scicat user. - """ +""" +These test_pyscicat do not use mocks and are designed to connect + to a v4 service for Scicat backend. You can run this easily +in docker-compose following the repo +https://github.com/SciCatProject/scicatlive. +You will also need to use one of the default user accounts or add +your own. - def test_client(self): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password=os.environ["SCICAT_PASSWORD"]) - self.assertIsInstance(sci_clie, ScicatClient) - print(sci_clie._token) +You will need to set environmental variables for +BASE_URL - the url of your scicat service e.g. http://localhost:3000/api/v3 +SCICAT_USER - the name of your scicat user. +SCICAT_PASSWORD - the password for your scicat user. +""" +def test_client(): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], + token=None, + username=os.environ["SCICAT_USER"], + password=os.environ["SCICAT_PASSWORD"]) + assert type(sci_clie) == ScicatClient - def test_upload_dataset(self): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password=os.environ["SCICAT_PASSWORD"]) - payload = RawDataset( - datasetName="a guide book", - path="/foo/bar", - size=42, - owner=os.environ["SCICAT_USER"], - ownerGroup="Magrateheans", - contactEmail="slartibartfast@magrathea.org", - creationLocation="magrathea", - creationTime= datetime.isoformat(datetime.now()), - instrumentId="earth", - proposalId="deepthought", - dataFormat="planet", - principalInvestigator="A. Mouse", - sourceFolder="/foo/bar", - scientificMetadata={"a": "field"}, - sampleId="gargleblaster", - accessGroups=[] - ) +def test_upload_dataset(): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], + token=None, username=os.environ["SCICAT_USER"], + password=os.environ["SCICAT_PASSWORD"]) - sci_clie.upload_new_dataset(payload) + payload = RawDataset( + datasetName="a guide book", + path="/foo/bar", + size=42, + owner=os.environ["SCICAT_USER"], + ownerGroup="Magrateheans", + contactEmail="slartibartfast@magrathea.org", + creationLocation="magrathea", + creationTime=datetime.isoformat(datetime.now()), + instrumentId="earth", + proposalId="deepthought", + dataFormat="planet", + principalInvestigator="A. Mouse", + sourceFolder="/foo/bar", + scientificMetadata={"a": "field"}, + sampleId="gargleblaster", + accessGroups=[] + ) - def test_get_dataset(self): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], password=os.environ["SCICAT_PASSWORD"]) - datasets = sci_clie.get_datasets({"ownerGroup": "Magratheans"}) - with self.subTest(dataset=datasets): - self.assertEqual(dataset["ownerGroup"], "Magratheans") + sci_clie.upload_new_dataset(payload) - def test_update_dataset(self): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], token=None, username=os.environ["SCICAT_USER"], - password=os.environ["SCICAT_PASSWORD"]) - - pid="PID.SAMPLE.PREFIX48a8f164-166a-4557-bafc-5a7362e39fe7" - payload= RawDataset( - size=142, - owner="slartibartfast", - ownerGroup="Magrateheans", - contactEmail="slartibartfast@magrathea.org", - creationLocation="magrathea", - creationTime=datetime.isoformat(datetime.now()), - instrumentId="earth", - proposalId="deepthought", - dataFormat="planet", - principalInvestigator="A. Mouse", - sourceFolder="/foo/bar", - scientificMetadata={"a": "field"}, - sampleId="gargleblaster", - accessGroups=["Vogons"] - ) - sci_clie.update_dataset(payload, pid) +def test_get_dataset(subtests): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], + token=None, + username=os.environ["SCICAT_USER"], + password=os.environ["SCICAT_PASSWORD"]) + datasets = sci_clie.get_datasets({"ownerGroup": "Magratheans"}) + for dataset in datasets: + with subtests.tests(dataset=dataset): + assert dataset["ownerGroup"] == "Magratheans" +def test_update_dataset(): + sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], + token=None, + username=os.environ["SCICAT_USER"], + password=os.environ["SCICAT_PASSWORD"]) + pid = "PID.SAMPLE.PREFIX48a8f164-166a-4557-bafc-5a7362e39fe7" + payload = RawDataset( + size=142, + owner="slartibartfast", + ownerGroup="Magrateheans", + contactEmail="slartibartfast@magrathea.org", + creationLocation="magrathea", + creationTime=datetime.isoformat(datetime.now()), + instrumentId="earth", + proposalId="deepthought", + dataFormat="planet", + principalInvestigator="A. Mouse", + sourceFolder="/foo/bar", + scientificMetadata={"a": "field"}, + sampleId="gargleblaster", + accessGroups=["Vogons"] + ) + sci_clie.update_dataset(payload, pid) From 3e16177c09c35af4cb8ba99f3dcb7a8b4eb5311f Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Mon, 20 Mar 2023 17:23:01 +0000 Subject: [PATCH 72/98] putting the python version to the lowest verions that is built against. The versioneer part of the set up config will set the version and does not need to be written in the metadata section --- pyscicat/_version.py | 388 +++++++++++++++++++++++++++---------------- setup.cfg | 5 +- setup.py | 6 +- 3 files changed, 252 insertions(+), 147 deletions(-) diff --git a/pyscicat/_version.py b/pyscicat/_version.py index 6977658..bae1847 100644 --- a/pyscicat/_version.py +++ b/pyscicat/_version.py @@ -1,11 +1,13 @@ + # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) +# This file is released into the public domain. +# Generated by versioneer-0.28 +# https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" @@ -14,6 +16,8 @@ import re import subprocess import sys +from typing import Callable, Dict +import functools def get_keywords(): @@ -51,40 +55,44 @@ class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" -LONG_VERSION_PY = {} -HANDLERS = {} +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - + """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f - return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): """Call the given command(s).""" assert isinstance(commands, list) - p = None - for c in commands: + process = None + + popen_kwargs = {} + if sys.platform == "win32": + # This hides the console window if pythonw.exe is used + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + popen_kwargs["startupinfo"] = startupinfo + + for command in commands: try: - dispcmd = str([c] + args) + dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None), **popen_kwargs) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -96,15 +104,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= if verbose: print("unable to find command, tried %s" % (commands,)) return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode + return None, process.returncode + return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): @@ -116,25 +122,18 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): """ rootdirs = [] - for i in range(3): + for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + rootdirs.append(root) + root = os.path.dirname(root) # up a level if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -147,22 +146,21 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: pass return keywords @@ -170,10 +168,14 @@ def git_get_keywords(versionfile_abs): @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because @@ -186,11 +188,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -199,7 +201,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) + tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -207,30 +209,28 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] + r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue if verbose: print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): +def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* @@ -241,7 +241,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) + # GIT_DIR can interfere with correct operation of Versioneer. + # It may be intended to be passed to the Versioneer-versioned project, + # but that should not change where we get our version from. + env = os.environ.copy() + env.pop("GIT_DIR", None) + runner = functools.partial(runner, env=env) + + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -249,24 +257,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) + describe_out, rc = runner(GITS, [ + "describe", "--tags", "--dirty", "--always", "--long", + "--match", f"{tag_prefix}[[:digit:]]*" + ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -276,6 +275,39 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out @@ -284,16 +316,17 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] + git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out + # unparsable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) return pieces # tag @@ -302,12 +335,10 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] + pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -318,13 +349,14 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits + out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) + pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -355,25 +387,74 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. +def render_pep440_branch(pieces): + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). Exceptions: - 1: no tags. 0.post.devDISTANCE + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver): + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces): + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] else: # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] + rendered = "0.post0.dev%d" % pieces["distance"] return rendered @@ -404,12 +485,41 @@ def render_pep440_post(pieces): return rendered +def render_pep440_post_branch(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. - Eexceptions: + Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: @@ -469,23 +579,25 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": @@ -495,13 +607,9 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} def get_versions(): @@ -515,7 +623,8 @@ def get_versions(): verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) except NotThisMethod: pass @@ -524,16 +633,13 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): + for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -547,10 +653,6 @@ def get_versions(): except NotThisMethod: pass - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} diff --git a/setup.cfg b/setup.cfg index e8e2dad..da3068b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,6 @@ tag_prefix = v [metadata] name = pyscicat -version = 1.0.0 description = a python API to communicate with the Scicat API long_description = file: README.md long_description_content_type = text/markdown @@ -20,7 +19,7 @@ license="BSD (3-clause)" classifiers= "Development Status :: 2 - Pre-Alpha" "Natural Language :: English" - "Programming Language :: Python :: 3" + "Programming Language :: Python :: 3.7" [options] @@ -29,7 +28,7 @@ install_requires = pydantic requests packages = find: -python_requires = >=3.0 +python_requires = >=3.7 [options.extras_require] diff --git a/setup.py b/setup.py index 6068493..1d0c607 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,7 @@ from setuptools import setup +import versioneer -setup() +setup( + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass() +) From 8b02d37c950e9ac7fbb949ef7fa963b5809c568f Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Wed, 19 Apr 2023 13:52:17 +0000 Subject: [PATCH 73/98] Removing the forward / before Datasets. If this is given in the endpoint then this is treated by urllib.parse like a full path. --- pyscicat/client.py | 6 +-- .../{tests_local.py => tests_integration.py} | 49 +++++++++++-------- 2 files changed, 31 insertions(+), 24 deletions(-) rename tests/test_pyscicat/{tests_local.py => tests_integration.py} (66%) diff --git a/pyscicat/client.py b/pyscicat/client.py index 9d3e05a..6ceeaa1 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -706,7 +706,7 @@ def datasets_get_many(self, filter_fields: Optional[dict] = None) -> Optional[di if not filter_fields: filter_fields = {} filter_fields = json.dumps(filter_fields) - endpoint = f'/Datasets/?filter={{"where":{filter_fields}}}' + endpoint = f'Datasets?filter={{"where":{filter_fields}}}' return self._call_endpoint( cmd="get", endpoint=endpoint, operation="datasets_get_many", allow_404=True ) @@ -879,7 +879,7 @@ def datasets_origdatablocks_get_one(self, pid: str) -> Optional[dict]: """ return self._call_endpoint( cmd="get", - endpoint=f"/Datasets/{quote_plus(pid)}/origdatablocks", + endpoint=f"Datasets/{quote_plus(pid)}/origdatablocks", operation="datasets_origdatablocks_get_one", allow_404=True, ) @@ -904,7 +904,7 @@ def datasets_delete(self, pid: str) -> Optional[dict]: """ return self._call_endpoint( cmd="delete", - endpoint=f"/Datasets/{quote_plus(pid)}", + endpoint=f"Datasets/{quote_plus(pid)}", operation="datasets_delete", allow_404=True, ) diff --git a/tests/test_pyscicat/tests_local.py b/tests/test_pyscicat/tests_integration.py similarity index 66% rename from tests/test_pyscicat/tests_local.py rename to tests/test_pyscicat/tests_integration.py index 95204bd..387fbe1 100644 --- a/tests/test_pyscicat/tests_local.py +++ b/tests/test_pyscicat/tests_integration.py @@ -1,7 +1,8 @@ from pyscicat.client import ScicatClient -from pyscicat.model import RawDataset +from pyscicat.model import RawDataset, Ownable from datetime import datetime import os +import requests """ @@ -18,51 +19,57 @@ SCICAT_PASSWORD - the password for your scicat user. """ - +sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], + token=None, + username=os.environ["SCICAT_USER"], + password=os.environ["SCICAT_PASSWORD"]) def test_client(): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], - token=None, - username=os.environ["SCICAT_USER"], - password=os.environ["SCICAT_PASSWORD"]) + assert type(sci_clie) == ScicatClient + def test_upload_dataset(): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], - token=None, username=os.environ["SCICAT_USER"], - password=os.environ["SCICAT_PASSWORD"]) + + ownable = Ownable(ownerGroup="ingestor", accessGroups=[]) payload = RawDataset( - datasetName="a guide book", + datasetName="a new guide book", path="/foo/bar", size=42, + packedSize=0, owner=os.environ["SCICAT_USER"], - ownerGroup="Magrateheans", contactEmail="slartibartfast@magrathea.org", - creationLocation="magrathea", + creationLocation="Magrathea", creationTime=datetime.isoformat(datetime.now()), instrumentId="earth", proposalId="deepthought", dataFormat="planet", principalInvestigator="A. Mouse", sourceFolder="/foo/bar", - scientificMetadata={"a": "field"}, + scientificMetadata={"type": "string", "value": {"a": "field"}}, sampleId="gargleblaster", - accessGroups=[] + type="raw", + ownerEmail="scicatingestor@your.site", + sourceFolderHost="s3.heartofgold.org", + endTime=datetime.isoformat(datetime.now()), + techniques=[], + numberOfFiles=0, + numberOfFilesArchived=0, + **ownable.dict() ) sci_clie.upload_new_dataset(payload) def test_get_dataset(subtests): - sci_clie = ScicatClient(base_url=os.environ["BASE_URL"], - token=None, - username=os.environ["SCICAT_USER"], - password=os.environ["SCICAT_PASSWORD"]) - datasets = sci_clie.get_datasets({"ownerGroup": "Magratheans"}) + + datasets = sci_clie.get_datasets({"ownerGroup": "ingestor"}) + + for dataset in datasets: - with subtests.tests(dataset=dataset): - assert dataset["ownerGroup"] == "Magratheans" + + assert dataset["ownerGroup"] == "ingestor" def test_update_dataset(): From 291858fbb078e94e2b5a281fc6cd898104fd4126 Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Wed, 19 Apr 2023 13:55:17 +0000 Subject: [PATCH 74/98] testing update dataset now that the getter methods work --- tests/test_pyscicat/tests_integration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pyscicat/tests_integration.py b/tests/test_pyscicat/tests_integration.py index 387fbe1..2386eb9 100644 --- a/tests/test_pyscicat/tests_integration.py +++ b/tests/test_pyscicat/tests_integration.py @@ -78,7 +78,8 @@ def test_update_dataset(): username=os.environ["SCICAT_USER"], password=os.environ["SCICAT_PASSWORD"]) - pid = "PID.SAMPLE.PREFIX48a8f164-166a-4557-bafc-5a7362e39fe7" + datasets = sci_clie.get_datasets({}) + pid = datasets[0]["pid"] payload = RawDataset( size=142, owner="slartibartfast", From e0371e3e531e6a99aee94f5af025a059b40c8601 Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Wed, 19 Apr 2023 14:23:06 +0000 Subject: [PATCH 75/98] move the integration tests out into their own package so they can be ignored by the current actions workflows. --- .github/workflows/testing.yml | 2 +- tests/tests_integration/__init__.py | 0 tests/{test_pyscicat => tests_integration}/tests_integration.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/tests_integration/__init__.py rename tests/{test_pyscicat => tests_integration}/tests_integration.py (100%) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 54fdadf..1435b5d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -45,5 +45,5 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail - coverage run -m pytest -v + coverage run -m pytest --ignore tests_integration -v coverage report diff --git a/tests/tests_integration/__init__.py b/tests/tests_integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_pyscicat/tests_integration.py b/tests/tests_integration/tests_integration.py similarity index 100% rename from tests/test_pyscicat/tests_integration.py rename to tests/tests_integration/tests_integration.py From b2336fae3f124f1ebbff9f9245659282ebd2c34c Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Fri, 21 Apr 2023 08:26:32 +0000 Subject: [PATCH 76/98] first attempt at an actions workflow to do the integration testing. --- .github/workflows/integration-testing.yml | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/integration-testing.yml diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml new file mode 100644 index 0000000..d2b1671 --- /dev/null +++ b/.github/workflows/integration-testing.yml @@ -0,0 +1,62 @@ +name: Integration tests V4 + +on: + pull_request: + +jobs: + container_job: + runs_on: ubuntu-latest + + services: + mongo: + image: scimongo + ports: 27107:27107 + scicat-backend: + image: ghcr/scicat-project/backend-next + ports: 3000:3000 + environmentVariables: + MONGODB_URI: mongodb://mongo:27017/scicat + EXPRESS_SESSION_SECRET: "${EXPRESS_SESSION_SECRET}" + JWT_SECRET: "${JWT_SECRET}" + PORT: 3000 + HTTP_MAX_REDIRECTS: 5 + HTTP_TIMEOUT: 5000 + JWT_EXPIRES_IN: 3600 + SITE: SAMPLE-SITE + PID_PREFIX: PID.SAMPLE.PREFIX + DOI_PREFIX: DOI.SAMPLE.PREFIX + METADATA_KEYS_RETURN_LIMIT: 100 + METADATA_PARENT_INSTANCES_RETURN_LIMIT: 100 + ADMIN_GROUPS: admin,ingestor + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install + shell: bash -l {0} + run: source continuous_integration/scripts/install.sh + + - name: Install test requirements + shell: bash -l {0} + run: | + set -vxeuo pipefail + python -m pip install -r requirements-dev.txt + python -m pip list + + - name: Lint with flake8 + shell: bash -l {0} + run: | + set -vxeuo pipefail + python -m flake8 + + - name: Test with pytest + shell: bash -l {0} + run: | + set -vxeuo pipefail + coverage run -m pytest -k tests_integration -v + coverage report \ No newline at end of file From dddf66474a79ce82b30a10f0d7c591ace0200e1d Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Fri, 21 Apr 2023 08:37:00 +0000 Subject: [PATCH 77/98] changing linting and adding steps --- .github/workflows/integration-testing.yml | 31 ++++++++--------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index d2b1671..5a1b9cb 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -2,6 +2,8 @@ name: Integration tests V4 on: pull_request: + branches: + - main jobs: container_job: @@ -10,10 +12,12 @@ jobs: services: mongo: image: scimongo - ports: 27107:27107 + ports: + - 27107:27107 scicat-backend: image: ghcr/scicat-project/backend-next - ports: 3000:3000 + ports: + - 3000:3000 environmentVariables: MONGODB_URI: mongodb://mongo:27017/scicat EXPRESS_SESSION_SECRET: "${EXPRESS_SESSION_SECRET}" @@ -31,32 +35,19 @@ jobs: steps: - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install - shell: bash -l {0} - run: source continuous_integration/scripts/install.sh + - run: source continuous_integration/scripts/install.sh + - - name: Install test requirements - shell: bash -l {0} - run: | + - run: | set -vxeuo pipefail python -m pip install -r requirements-dev.txt python -m pip list - - name: Lint with flake8 - shell: bash -l {0} - run: | - set -vxeuo pipefail - python -m flake8 - - - name: Test with pytest - shell: bash -l {0} - run: | + - run: | set -vxeuo pipefail coverage run -m pytest -k tests_integration -v coverage report \ No newline at end of file From f6770543eeded9d5457848fcdefb7068a0ba9b59 Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Fri, 21 Apr 2023 08:38:15 +0000 Subject: [PATCH 78/98] changing linting and adding steps --- .github/workflows/integration-testing.yml | 8 ++++---- tests/test_hdf5/__init__.py | 0 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 tests/test_hdf5/__init__.py diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index 5a1b9cb..a4d9274 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -7,18 +7,18 @@ on: jobs: container_job: - runs_on: ubuntu-latest + runs-on: ubuntu-latest services: mongo: - image: scimongo + image: mongo ports: - 27107:27107 scicat-backend: image: ghcr/scicat-project/backend-next ports: - 3000:3000 - environmentVariables: + env: MONGODB_URI: mongodb://mongo:27017/scicat EXPRESS_SESSION_SECRET: "${EXPRESS_SESSION_SECRET}" JWT_SECRET: "${JWT_SECRET}" @@ -33,7 +33,7 @@ jobs: METADATA_PARENT_INSTANCES_RETURN_LIMIT: 100 ADMIN_GROUPS: admin,ingestor - steps: + steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: diff --git a/tests/test_hdf5/__init__.py b/tests/test_hdf5/__init__.py new file mode 100644 index 0000000..e69de29 From 6b8474149fc5eeb08803ec5d0c9e6539ca17134f Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Fri, 21 Apr 2023 08:54:13 +0000 Subject: [PATCH 79/98] correct name for the scicat backend service --- .github/workflows/integration-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index a4d9274..ed9cb7f 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -15,7 +15,7 @@ jobs: ports: - 27107:27107 scicat-backend: - image: ghcr/scicat-project/backend-next + image: ghcr.io/scicatproject/scicat-backend-next:stable ports: - 3000:3000 env: From 88343de9d17b94be9457acdb26956a5999f6a3ce Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Fri, 21 Apr 2023 09:35:57 +0000 Subject: [PATCH 80/98] adding environmental variables --- .github/workflows/integration-testing.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index ed9cb7f..24ad374 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -50,4 +50,8 @@ jobs: - run: | set -vxeuo pipefail coverage run -m pytest -k tests_integration -v - coverage report \ No newline at end of file + coverage report + env: + BASE_URL: http://localhost:3000/api/v3 + SCICAT_USER: ingestor + SCICAT_PASSWORD: aman \ No newline at end of file From 452df367f555ed6b8a73b2a0ea317873ba11cd7d Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Fri, 21 Apr 2023 09:46:49 +0000 Subject: [PATCH 81/98] remove subtests dependency --- tests/tests_integration/tests_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_integration/tests_integration.py b/tests/tests_integration/tests_integration.py index 2386eb9..d878848 100644 --- a/tests/tests_integration/tests_integration.py +++ b/tests/tests_integration/tests_integration.py @@ -62,7 +62,7 @@ def test_upload_dataset(): sci_clie.upload_new_dataset(payload) -def test_get_dataset(subtests): +def test_get_dataset(): datasets = sci_clie.get_datasets({"ownerGroup": "ingestor"}) From ecd84c4b20f18028366429571da40ce370a3de0e Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Thu, 11 May 2023 12:22:47 +0000 Subject: [PATCH 82/98] solving merge --- .github/workflows/integration-testing.yml | 4 +++- requirements-hdf5.txt | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 requirements-hdf5.txt diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index 24ad374..2aafbf0 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -44,11 +44,13 @@ jobs: - run: | set -vxeuo pipefail - python -m pip install -r requirements-dev.txt + python -m pip install . + python -m pip install .[dev] python -m pip list - run: | set -vxeuo pipefail + coverage -m pytest -k tests_integration -v coverage run -m pytest -k tests_integration -v coverage report env: diff --git a/requirements-hdf5.txt b/requirements-hdf5.txt deleted file mode 100644 index d03a9f6..0000000 --- a/requirements-hdf5.txt +++ /dev/null @@ -1,2 +0,0 @@ -hdf5plugin -h5py From b5e7ffba0d482edb5720e28bb5d1683e98b796f5 Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Thu, 11 May 2023 12:26:10 +0000 Subject: [PATCH 83/98] removing methods from v4 --- pyscicat/client.py | 145 +-------------------------------------------- setup.cfg | 4 ++ 2 files changed, 5 insertions(+), 144 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index 6ceeaa1..3332d92 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -12,6 +12,7 @@ from pydantic import BaseModel import requests + from pyscicat.model import ( Attachment, Datablock, @@ -132,44 +133,6 @@ def _call_endpoint( ) return result - def datasets_replace(self, dataset: Dataset) -> str: - """ - Create a new dataset or update an existing one - This function was renamed. - It is still accessible with the original name for backward compatibility - The original names were upload_dataset replace_datasets - This function is obsolete and it will be remove in next relesases - - - Parameters - ---------- - dataset : Dataset - Dataset to create or update - - Returns - ------- - str - pid of the dataset - """ - - if isinstance(dataset, RawDataset): - dataset_url = "RawDataSets/replaceOrCreate" - elif isinstance(dataset, DerivedDataset): - dataset_url = "DerivedDatasets/replaceOrCreate" - else: - raise TypeError( - "Dataset type not recognized (not Derived or Raw dataset instances)" - ) - return self._call_endpoint( - cmd="post", endpoint=dataset_url, data=dataset, operation="datasets_replace" - ).get("pid") - - """ - Upload or create a new dataset - Original name, kept for for backward compatibility - """ - upload_dataset = datasets_replace - replace_dataset = datasets_replace def datasets_create(self, dataset: Dataset) -> str: """ @@ -205,72 +168,7 @@ def datasets_create(self, dataset: Dataset) -> str: upload_new_dataset = datasets_create create_dataset = datasets_create - def datasets_raw_replace(self, dataset: Dataset) -> str: - """ - Create a new raw dataset or update an existing one - This function was renamed. - It is still accessible with the original name for backward compatibility - The original names were repalce_raw_dataset and upload_raw_dataset - This function is obsolete and it will be removed in future releases - - Parameters - ---------- - dataset : Dataset - Dataset to load - - Returns - ------- - str - pid (or unique identifier) of the newly created dataset - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - return self._call_endpoint( - cmd="post", - endpoint="RawDataSets/replaceOrCreate", - data=dataset, - operation="datasets_raw_replace", - ).get("pid") - - """ - Upload a raw dataset - Original name, kept for for backward compatibility - """ - upload_raw_dataset = datasets_raw_replace - replace_raw_dataset = datasets_raw_replace - - def datasets_derived_replace(self, dataset: Dataset) -> str: - """ - Create a new derived dataset or update an existing one - This function was renamed. - It is still accessible with the original name for backward compatibility - The original names were replace_derived_dataset and upload_derived_dataset - - - Parameters - ---------- - dataset : Dataset - Dataset to upload - - Returns - ------- - str - pid (or unique identifier) of the newly created dataset - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - return self._call_endpoint( - cmd="post", - endpoint="DerivedDataSets/replaceOrCreate", - data=dataset, - operation="datasets_derived_replace", - ).get("pid") def datasets_update(self, dataset: Dataset, pid: str) -> str: """Updates an existing dataset @@ -308,47 +206,6 @@ def datasets_update(self, dataset: Dataset, pid: str) -> str: """ update_dataset = datasets_update - def datasets_datablock_create( - self, datablock: Datablock, datasetType: str = "RawDatasets" - ) -> dict: - """ - Create a new datablock for a dataset. - The dataset can be both Raw or Derived. - It is still accessible with the original name for backward compatibility - The original names were create_dataset_datablock and upload_datablock - This function is obsolete and will be removed in future releases - Function datasets_origdatablock_create should be used. - - Parameters - ---------- - datablock : Datablock - Datablock to upload - - Returns - ------- - datablock : Datablock - The created Datablock with id - - Raises - ------ - ScicatCommError - Raises if a non-20x message is returned - """ - endpoint = f"{datasetType}/{quote_plus(datablock.datasetId)}/origdatablocks" - return self._call_endpoint( - cmd="post", - endpoint=endpoint, - data=datablock, - operation="datasets_datablock_create", - ) - - """ - Upload a Datablock - Original name, kept for for backward compatibility - """ - upload_datablock = datasets_datablock_create - create_dataset_datablock = datasets_datablock_create - def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: """ Create a new SciCat Dataset OrigDatablock diff --git a/setup.cfg b/setup.cfg index da3068b..c67ba5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,10 @@ include_package_data = True install_requires = pydantic requests +<<<<<<< HEAD +======= + +>>>>>>> b373c11... remove deprecation methods and dependency. We have decided to make a clean break with pyscicat versions. If the method does not exist in the new backend it will be deleted from pyscicat. packages = find: python_requires = >=3.7 From d3ff9d13c7c8c52fb580adb425f1816d2fe7d54f Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Thu, 11 May 2023 12:30:54 +0000 Subject: [PATCH 84/98] remove erroneous line in coverage stage of integration-testing.yml --- .github/workflows/integration-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index 2aafbf0..44bfdc9 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -46,11 +46,11 @@ jobs: set -vxeuo pipefail python -m pip install . python -m pip install .[dev] + python -m pip install .[hdf5] python -m pip list - run: | set -vxeuo pipefail - coverage -m pytest -k tests_integration -v coverage run -m pytest -k tests_integration -v coverage report env: From 8350d3c755acbd60b98bba2039ca12b5dbb7966a Mon Sep 17 00:00:00 2001 From: lashemilt Date: Fri, 2 Jun 2023 15:56:35 +0000 Subject: [PATCH 85/98] replace pip install from requirements-dev.txt with the .[dev] option from the setup config. --- .github/workflows/publish-documentation.yml | 2 +- .github/workflows/testing.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-documentation.yml b/.github/workflows/publish-documentation.yml index 78f4544..5a1571f 100644 --- a/.github/workflows/publish-documentation.yml +++ b/.github/workflows/publish-documentation.yml @@ -37,7 +37,7 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail - python -m pip install -r requirements-dev.txt + python -m pip install .[dev] python -m pip list - name: Build Docs shell: bash -l {0} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1435b5d..e5298cd 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -32,7 +32,7 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail - python -m pip install -r requirements-dev.txt + python -m pip install .[dev] python -m pip list - name: Lint with flake8 From 420f88bdc21b2bef3601763367e802ddfb0ec957 Mon Sep 17 00:00:00 2001 From: lashemilt Date: Tue, 6 Jun 2023 08:02:37 +0000 Subject: [PATCH 86/98] updating the tests mocks to be inline with the new backend --- pyscicat/client.py | 2 +- requirements-dev.txt | 20 -------------------- requirements.txt | 2 -- setup.cfg | 4 ---- tests/test_pyscicat/test_client.py | 10 +++++----- 5 files changed, 6 insertions(+), 32 deletions(-) delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt diff --git a/pyscicat/client.py b/pyscicat/client.py index 3332d92..9d41cc8 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -245,7 +245,7 @@ def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: create_dataset_origdatablock = datasets_origdatablock_create def datasets_attachment_create( - self, attachment: Attachment, datasetType: str = "RawDatasets" + self, attachment: Attachment, datasetType: str = "Datasets" ) -> dict: """ Create a new Attachment for a dataset. diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 7c4c01d..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,20 +0,0 @@ -# These are required for developing the package (running the tests, building -# the documentation) but not necessarily required for _using_ it. -codecov -coverage -flake8 -pytest -sphinx -twine -black -requests_mock -# These are dependencies of various sphinx extensions for documentation. -ipython -matplotlib -mistune <2.0.0 # temporary while sphinx sorts this out -myst-parser -numpydoc -sphinx-click -sphinx-copybutton -sphinxcontrib.openapi -sphinx_rtd_theme diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 903705e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pydantic -requests diff --git a/setup.cfg b/setup.cfg index c67ba5d..9de6a05 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,11 +27,7 @@ include_package_data = True install_requires = pydantic requests -<<<<<<< HEAD -======= ->>>>>>> b373c11... remove deprecation methods and dependency. We have decided to make a clean break with pyscicat versions. If the method does not exist in the new backend it will be deleted from pyscicat. -packages = find: python_requires = >=3.7 diff --git a/tests/test_pyscicat/test_client.py b/tests/test_pyscicat/test_client.py index c165007..aa5c308 100644 --- a/tests/test_pyscicat/test_client.py +++ b/tests/test_pyscicat/test_client.py @@ -49,15 +49,15 @@ def add_mock_requests(mock_request): json={"pid": "42"}, ) mock_request.post( - local_url + "RawDatasets/42/origdatablocks", + local_url + "Datasets/42/origdatablocks", json={"response": "random"}, ) mock_request.post( - local_url + "RawDatasets/42/attachments", + local_url + "Datasets/42/attachments", json={"response": "random"}, ) - mock_request.post(local_url + "Datasets", json={"pid": "17"}) + mock_request.post(local_url + "Datasets", json={"pid": "42"}) def test_scicat_ingest(): @@ -128,7 +128,7 @@ def test_scicat_ingest(): sampleId="gargleblaster", **ownable.dict() ) - dataset_id = scicat.upload_raw_dataset(dataset) + dataset_id = scicat.upload_new_dataset(dataset) assert dataset_id == "42" # Update record @@ -145,7 +145,7 @@ def test_scicat_ingest(): dataFileList=[data_file], **ownable.dict() ) - scicat.upload_datablock(data_block) + scicat.upload_dataset_origdatablock(data_block) # Attachment attachment = Attachment( From 75c1f9a331654177d06a84bbdaccc2ee77332892 Mon Sep 17 00:00:00 2001 From: lashemilt Date: Tue, 6 Jun 2023 08:05:56 +0000 Subject: [PATCH 87/98] liniting --- pyscicat/__init__.py | 2 -- pyscicat/client.py | 6 ------ tests/tests_integration/tests_integration.py | 10 +++------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/pyscicat/__init__.py b/pyscicat/__init__.py index 32ac1d7..80edaf0 100644 --- a/pyscicat/__init__.py +++ b/pyscicat/__init__.py @@ -1,5 +1,3 @@ -from . import hdf5 -from . import ingest from ._version import get_versions __version__ = get_versions()["version"] diff --git a/pyscicat/client.py b/pyscicat/client.py index 9d41cc8..f43a52d 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -15,13 +15,10 @@ from pyscicat.model import ( Attachment, - Datablock, Dataset, - DerivedDataset, Instrument, OrigDatablock, Proposal, - RawDataset, Sample, ) @@ -133,7 +130,6 @@ def _call_endpoint( ) return result - def datasets_create(self, dataset: Dataset) -> str: """ Upload a new dataset. Uses the generic dataset endpoint. @@ -168,8 +164,6 @@ def datasets_create(self, dataset: Dataset) -> str: upload_new_dataset = datasets_create create_dataset = datasets_create - - def datasets_update(self, dataset: Dataset, pid: str) -> str: """Updates an existing dataset This function was renamed. diff --git a/tests/tests_integration/tests_integration.py b/tests/tests_integration/tests_integration.py index d878848..79c8168 100644 --- a/tests/tests_integration/tests_integration.py +++ b/tests/tests_integration/tests_integration.py @@ -2,7 +2,6 @@ from pyscicat.model import RawDataset, Ownable from datetime import datetime import os -import requests """ @@ -23,15 +22,14 @@ token=None, username=os.environ["SCICAT_USER"], password=os.environ["SCICAT_PASSWORD"]) + + def test_client(): assert type(sci_clie) == ScicatClient - def test_upload_dataset(): - - ownable = Ownable(ownerGroup="ingestor", accessGroups=[]) payload = RawDataset( datasetName="a new guide book", @@ -66,10 +64,8 @@ def test_get_dataset(): datasets = sci_clie.get_datasets({"ownerGroup": "ingestor"}) - for dataset in datasets: - - assert dataset["ownerGroup"] == "ingestor" + assert dataset["ownerGroup"] == "ingestor" def test_update_dataset(): From 55d0f3838466ae4d71e36b4b22dd238a49c56950 Mon Sep 17 00:00:00 2001 From: lashemilt Date: Tue, 6 Jun 2023 08:17:06 +0000 Subject: [PATCH 88/98] updating deprecated config - license_file to license_files --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9de6a05..09ddb0b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ long_description_content_type = text/markdown author = Dylan McReynolds author_email = "dmcreynolds@lbl.gov", url = "https://github.com/scicatproject/pyscicat" -license_file = LICENSE +license_files = LICENSE license="BSD (3-clause)" classifiers= "Development Status :: 2 - Pre-Alpha" From 6f13a4ebce3c1e02b7b468a0c8cc24b7d53fa397 Mon Sep 17 00:00:00 2001 From: lashemilt Date: Tue, 6 Jun 2023 08:27:18 +0000 Subject: [PATCH 89/98] excluding continuous_integration folder that is used in workflows but is not part of package --- .github/workflows/integration-testing.yml | 1 - .github/workflows/testing.yml | 1 + setup.cfg | 8 +++++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index 44bfdc9..2d289ed 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -46,7 +46,6 @@ jobs: set -vxeuo pipefail python -m pip install . python -m pip install .[dev] - python -m pip install .[hdf5] python -m pip list - run: | diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e5298cd..b71b1be 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -33,6 +33,7 @@ jobs: run: | set -vxeuo pipefail python -m pip install .[dev] + python -m pip install .[hdf5] python -m pip list - name: Lint with flake8 diff --git a/setup.cfg b/setup.cfg index 09ddb0b..cb38c22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ classifiers= [options] include_package_data = True +packages= find: install_requires = pydantic requests @@ -31,6 +32,7 @@ install_requires = python_requires = >=3.7 + [options.extras_require] hdf5 = hdf5plugin @@ -53,4 +55,8 @@ docs = sphinx-click sphinx-copybutton sphinxcontrib.openapi - sphinx_rtd_theme \ No newline at end of file + sphinx_rtd_theme + +[options.packages.find] +exclude = + continuous_integration From a66551b281a270ac908c5d512d06d6ec7ce77daa Mon Sep 17 00:00:00 2001 From: lashemilt Date: Tue, 6 Jun 2023 08:36:32 +0000 Subject: [PATCH 90/98] run just the integration tests --- .github/workflows/integration-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-testing.yml b/.github/workflows/integration-testing.yml index 2d289ed..a9accb2 100644 --- a/.github/workflows/integration-testing.yml +++ b/.github/workflows/integration-testing.yml @@ -50,7 +50,7 @@ jobs: - run: | set -vxeuo pipefail - coverage run -m pytest -k tests_integration -v + coverage run -m pytest tests/tests_integration/tests_integration.py coverage report env: BASE_URL: http://localhost:3000/api/v3 From 46a69f4d4b5ed3ae0488d3b7ea1e64bf4ac27f7c Mon Sep 17 00:00:00 2001 From: vpf26432 Date: Tue, 27 Jun 2023 12:29:55 +0000 Subject: [PATCH 91/98] removing quotation marks that causes pypi to fail --- setup.cfg | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index cb38c22..e87f52d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,14 +12,14 @@ description = a python API to communicate with the Scicat API long_description = file: README.md long_description_content_type = text/markdown author = Dylan McReynolds -author_email = "dmcreynolds@lbl.gov", -url = "https://github.com/scicatproject/pyscicat" +author_email = dmcreynolds@lbl.gov +url = https://github.com/scicatproject/pyscicat license_files = LICENSE -license="BSD (3-clause)" +license=BSD (3-clause) classifiers= - "Development Status :: 2 - Pre-Alpha" - "Natural Language :: English" - "Programming Language :: Python :: 3.7" + Development Status :: 2 - Pre-Alpha + Natural Language :: English + Programming Language :: Python :: 3.7 [options] From 70a40322cb8694f83068675d5d8a64b524526319 Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Wed, 2 Aug 2023 16:38:42 +0200 Subject: [PATCH 92/98] add default value None for all optional parameters --- pyscicat/model.py | 142 +++++++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/pyscicat/model.py b/pyscicat/model.py index 837e9bf..efb9919 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -16,18 +16,18 @@ class DatasetType(str, enum.Enum): class MongoQueryable(BaseModel): """Many objects in SciCat are mongo queryable""" - createdBy: Optional[str] - updatedBy: Optional[str] - updatedAt: Optional[str] - createdAt: Optional[str] + createdBy: Optional[str] = None + updatedBy: Optional[str] = None + updatedAt: Optional[str] = None + createdAt: Optional[str] = None class Ownable(MongoQueryable): """Many objects in SciCat are ownable""" ownerGroup: str - accessGroups: Optional[List[str]] - instrumentGroup: Optional[str] + accessGroups: Optional[List[str]] = None + instrumentGroup: Optional[str] = None class User(BaseModel): @@ -47,19 +47,19 @@ class Proposal(Ownable): """ proposalId: str - pi_email: Optional[str] - pi_firstname: Optional[str] - pi_lastname: Optional[str] + pi_email: Optional[str] = None + pi_firstname: Optional[str] = None + pi_lastname: Optional[str] = None email: str - firstname: Optional[str] - lastname: Optional[str] - title: Optional[str] # required in next backend version - abstract: Optional[str] - startTime: Optional[str] - endTime: Optional[str] + firstname: Optional[str] = None + lastname: Optional[str] = None + title: Optional[str] = None # required in next backend version + abstract: Optional[str] = None + startTime: Optional[str] = None + endTime: Optional[str] = None MeasurementPeriodList: Optional[ List[dict] - ] # may need updating with the measurement period model + ] = None # may need updating with the measurement period model class Sample(Ownable): @@ -68,10 +68,10 @@ class Sample(Ownable): Raw datasets should be linked to such sample definitions. """ - sampleId: Optional[str] - owner: Optional[str] - description: Optional[str] - sampleCharacteristics: Optional[dict] + sampleId: Optional[str] = None + owner: Optional[str] = None + description: Optional[str] = None + sampleCharacteristics: Optional[dict] = None isPublished: bool = False @@ -83,15 +83,15 @@ class Job(MongoQueryable): track of analysis jobs e.g. for automated analysis workflows """ - id: Optional[str] + id: Optional[str] = None emailJobInitiator: str type: str - creationTime: Optional[str] # not sure yet which ones are optional or not. - executionTime: Optional[str] - jobParams: Optional[dict] - jobStatusMessage: Optional[str] - datasetList: Optional[dict] # documentation says dict, but should maybe be list? - jobResultObject: Optional[dict] # ibid. + creationTime: Optional[str] = None # not sure yet which ones are optional or not. + executionTime: Optional[str] = None + jobParams: Optional[dict] = None + jobStatusMessage: Optional[str] = None + datasetList: Optional[dict] = None # documentation says dict, but should maybe be list? + jobResultObject: Optional[dict] = None # ibid. class Instrument(MongoQueryable): @@ -99,9 +99,9 @@ class Instrument(MongoQueryable): Instrument class, most of this is flexibly definable in customMetadata """ - pid: Optional[str] + pid: Optional[str] = None name: str - customMetadata: Optional[dict] + customMetadata: Optional[dict] = None class Dataset(Ownable): @@ -109,32 +109,32 @@ class Dataset(Ownable): A dataset in SciCat, base class for derived and raw datasets """ - pid: Optional[str] - classification: Optional[str] + pid: Optional[str] = None + classification: Optional[str] = None contactEmail: str creationTime: str # datetime - datasetName: Optional[str] - description: Optional[str] - history: Optional[List[dict]] # list of foreigh key ids to the Messages table - instrumentId: Optional[str] + datasetName: Optional[str] = None + description: Optional[str] = None + history: Optional[List[dict]] = None # list of foreigh key ids to the Messages table + instrumentId: Optional[str] = None isPublished: Optional[bool] = False - keywords: Optional[List[str]] - license: Optional[str] - numberOfFiles: Optional[int] - numberOfFilesArchived: Optional[int] - orcidOfOwner: Optional[str] - packedSize: Optional[int] + keywords: Optional[List[str]] = None + license: Optional[str] = None + numberOfFiles: Optional[int] = None + numberOfFilesArchived: Optional[int] = None + orcidOfOwner: Optional[str] = None + packedSize: Optional[int] = None owner: str - ownerEmail: Optional[str] - sharedWith: Optional[List[str]] - size: Optional[int] + ownerEmail: Optional[str] = None + sharedWith: Optional[List[str]] = None + size: Optional[int] = None sourceFolder: str - sourceFolderHost: Optional[str] - techniques: Optional[List[dict]] # with {'pid':pid, 'name': name} as entries + sourceFolderHost: Optional[str] = None + techniques: Optional[List[dict]] = None # with {'pid':pid, 'name': name} as entries type: DatasetType - validationStatus: Optional[str] - version: Optional[str] - scientificMetadata: Optional[Dict] + validationStatus: Optional[str] = None + version: Optional[str] = None + scientificMetadata: Optional[Dict] = None class RawDataset(Dataset): @@ -142,13 +142,13 @@ class RawDataset(Dataset): Raw datasets from which derived datasets are... derived. """ - principalInvestigator: Optional[str] - creationLocation: Optional[str] + principalInvestigator: Optional[str] = None + creationLocation: Optional[str] = None type: DatasetType = DatasetType.raw - dataFormat: Optional[str] - endTime: Optional[str] # datetime - sampleId: Optional[str] - proposalId: Optional[str] + dataFormat: Optional[str] = None + endTime: Optional[str] = None # datetime + sampleId: Optional[str] = None + proposalId: Optional[str] = None class DerivedDataset(Dataset): @@ -159,8 +159,8 @@ class DerivedDataset(Dataset): investigator: str inputDatasets: List[str] usedSoftware: List[str] - jobParameters: Optional[dict] - jobLogData: Optional[str] + jobParameters: Optional[dict] = None + jobLogData: Optional[str] = None type: DatasetType = DatasetType.derived @@ -173,8 +173,8 @@ class DataFile(MongoQueryable): path: str size: int - time: Optional[str] - chk: Optional[str] + time: Optional[str] = None + chk: Optional[str] = None uid: Optional[str] = None gid: Optional[str] = None perm: Optional[str] = None @@ -185,13 +185,13 @@ class Datablock(Ownable): A Datablock maps between a Dataset and contains DataFiles """ - id: Optional[str] + id: Optional[str] = None # archiveId: str = None listed in catamel model, but comes back invalid? size: int - packedSize: Optional[int] - chkAlg: Optional[int] + packedSize: Optional[int] = None + chkAlg: Optional[int] = None version: str = None - instrumentGroup: Optional[str] + instrumentGroup: Optional[str] = None dataFileList: List[DataFile] datasetId: str @@ -201,10 +201,10 @@ class OrigDatablock(Ownable): An Original Datablock maps between a Dataset and contains DataFiles """ - id: Optional[str] + id: Optional[str] = None # archiveId: str = None listed in catamel model, but comes back invalid? size: int - instrumentGroup: Optional[str] + instrumentGroup: Optional[str] = None dataFileList: List[DataFile] datasetId: str @@ -214,9 +214,9 @@ class Attachment(Ownable): Attachments can be any base64 encoded string...thumbnails are attachments """ - id: Optional[str] + id: Optional[str] = None thumbnail: str - caption: Optional[str] + caption: Optional[str] = None datasetId: str @@ -231,17 +231,17 @@ class PublishedData: publisher: str publicationYear: int title: str - url: Optional[str] + url: Optional[str] = None abstract: str dataDescription: str resourceType: str - numberOfFiles: Optional[int] - sizeOfArchive: Optional[int] + numberOfFiles: Optional[int] = None + sizeOfArchive: Optional[int] = None pidArray: List[str] authors: List[str] registeredTime: str status: str - thumbnail: Optional[str] + thumbnail: Optional[str] = None createdBy: str updatedBy: str createdAt: str From a30a4ac9b17a829e97e114be45ccf23d68e3e3f6 Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Wed, 2 Aug 2023 17:40:22 +0200 Subject: [PATCH 93/98] update to flake8-6.1 --- tests/tests_integration/tests_integration.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tests_integration/tests_integration.py b/tests/tests_integration/tests_integration.py index 79c8168..86ad83c 100644 --- a/tests/tests_integration/tests_integration.py +++ b/tests/tests_integration/tests_integration.py @@ -25,8 +25,7 @@ def test_client(): - - assert type(sci_clie) == ScicatClient + assert type(sci_clie) == ScicatClient # noqa: E721 def test_upload_dataset(): From 3628eb9d8ca0025adf836dffc7af72f768803a9f Mon Sep 17 00:00:00 2001 From: Martin Voigt Date: Wed, 2 Aug 2023 18:45:18 +0200 Subject: [PATCH 94/98] pydantic v2 no longer converts int to str automatically --- tests/test_pyscicat/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pyscicat/test_client.py b/tests/test_pyscicat/test_client.py index aa5c308..46584f1 100644 --- a/tests/test_pyscicat/test_client.py +++ b/tests/test_pyscicat/test_client.py @@ -140,7 +140,7 @@ def test_scicat_ingest(): data_file = DataFile(path="/foo/bar", size=42) data_block = Datablock( size=42, - version=1, + version="1", datasetId=dataset_id, dataFileList=[data_file], **ownable.dict() From c3cbb2fa8102cbddb548fd165d29be6446e4fe63 Mon Sep 17 00:00:00 2001 From: Jeffrey Fulmer Gardner Date: Fri, 25 Aug 2023 14:59:07 -0700 Subject: [PATCH 95/98] This commit makes a small change with the client in order to better support the current routes available on the newest version(v4.x) of the scicat backend. It also makes a small change to the OrigDatablock to support what is required in the new backend. --- pyscicat/client.py | 2 +- pyscicat/model.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index f43a52d..a8f55bc 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -239,7 +239,7 @@ def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: create_dataset_origdatablock = datasets_origdatablock_create def datasets_attachment_create( - self, attachment: Attachment, datasetType: str = "Datasets" + self, attachment: Attachment, datasetType: str = "Datasets" #V4.x of Scicat's backend does not have a route for RawDatasets so if datasetType is not provided it will cause 502 ) -> dict: """ Create a new Attachment for a dataset. diff --git a/pyscicat/model.py b/pyscicat/model.py index efb9919..5b70a00 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -202,7 +202,7 @@ class OrigDatablock(Ownable): """ id: Optional[str] = None - # archiveId: str = None listed in catamel model, but comes back invalid? + chkAlg :Optional[str] = None size: int instrumentGroup: Optional[str] = None dataFileList: List[DataFile] @@ -245,4 +245,4 @@ class PublishedData: createdBy: str updatedBy: str createdAt: str - updatedAt: str + updatedAt: str \ No newline at end of file From 23e56fd2ea44607f996cb05758dfc19cfcf88077 Mon Sep 17 00:00:00 2001 From: Jeffrey Fulmer Gardner Date: Tue, 12 Sep 2023 08:25:23 -0700 Subject: [PATCH 96/98] removed unnecessary comment about datasetType --- pyscicat/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index a8f55bc..f43a52d 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -239,7 +239,7 @@ def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict: create_dataset_origdatablock = datasets_origdatablock_create def datasets_attachment_create( - self, attachment: Attachment, datasetType: str = "Datasets" #V4.x of Scicat's backend does not have a route for RawDatasets so if datasetType is not provided it will cause 502 + self, attachment: Attachment, datasetType: str = "Datasets" ) -> dict: """ Create a new Attachment for a dataset. From d0c2a5772019eda7a323167ee23c795b4b872f16 Mon Sep 17 00:00:00 2001 From: Jeffrey Fulmer Gardner Date: Tue, 12 Sep 2023 08:34:46 -0700 Subject: [PATCH 97/98] fixed small comment error about which dataset is default --- pyscicat/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyscicat/client.py b/pyscicat/client.py index f43a52d..1a06ac5 100644 --- a/pyscicat/client.py +++ b/pyscicat/client.py @@ -255,7 +255,7 @@ def datasets_attachment_create( Attachment to upload datasetType : str - Type of dataset to upload to, default is `RawDatasets` + Type of dataset to upload to, default is `Datasets` Raises ------ ScicatCommError From b7170c2fc7a127efb1d69fc88a157529e44e08e1 Mon Sep 17 00:00:00 2001 From: Jeffrey Date: Wed, 13 Sep 2023 11:29:54 -0700 Subject: [PATCH 98/98] Update model.py ownerGroup will now longer be required by the pydantic model and because the backend will not accept it --- pyscicat/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyscicat/model.py b/pyscicat/model.py index 5b70a00..f7251be 100644 --- a/pyscicat/model.py +++ b/pyscicat/model.py @@ -25,7 +25,7 @@ class MongoQueryable(BaseModel): class Ownable(MongoQueryable): """Many objects in SciCat are ownable""" - ownerGroup: str + ownerGroup: Optional[str] = None accessGroups: Optional[List[str]] = None instrumentGroup: Optional[str] = None @@ -245,4 +245,4 @@ class PublishedData: createdBy: str updatedBy: str createdAt: str - updatedAt: str \ No newline at end of file + updatedAt: str