From ad4dd2e6ef901335bad23e7d106ab62c7c749b08 Mon Sep 17 00:00:00 2001 From: Sam Meluch Date: Tue, 30 Apr 2024 13:56:44 -0700 Subject: [PATCH] Add virtual repo snapshot feature to tdnf --- client/config.c | 5 + client/prototypes.h | 9 +- client/repo.c | 50 +- client/repolist.c | 66 ++ common/config.h | 1 + etc/bash_completion.d/tdnf-completion.bash | 7 +- include/tdnferror.h | 9 + include/tdnftypes.h | 3 + solv/defines.h | 29 + solv/includes.h | 6 +- solv/prototypes.h | 3 +- solv/tdnfrepo.c | 783 ++++++++++++++++++++- tools/cli/lib/help.c | 2 + tools/cli/lib/parseargs.c | 2 + 14 files changed, 958 insertions(+), 17 deletions(-) diff --git a/client/config.c b/client/config.c index 81eefeb4..c346037c 100644 --- a/client/config.c +++ b/client/config.c @@ -100,6 +100,10 @@ TDNFConfigFromCnfTree(PTDNF_CONF pConf, struct cnfnode *cn_top) { pConf->nInstallOnlyLimit = strtoi(cn->value); } + else if (strcmp(cn->name, TDNF_CONF_KEY_SNAPSHOT_TIME) == 0) + { + pConf->pszSnapshotTime = strdup(cn->value); + } else if (strcmp(cn->name, TDNF_CONF_KEY_CLEAN_REQ_ON_REMOVE) == 0) { pConf->nCleanRequirementsOnRemove = isTrue(cn->value); @@ -255,6 +259,7 @@ TDNFReadConfig( pConf->nOpenMax = TDNF_CONF_DEFAULT_OPENMAX; pConf->nInstallOnlyLimit = TDNF_CONF_DEFAULT_INSTALLONLY_LIMIT; pConf->nSSLVerify = TDNF_CONF_DEFAULT_SSLVERIFY; + pConf->pszSnapshotTime = NULL; register_ini(NULL); mod_ini = find_cnfmodule("ini"); diff --git a/client/prototypes.h b/client/prototypes.h index ee632ed9..8bcc5518 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -578,7 +578,8 @@ uint32_t TDNFInitRepoFromMetadata( Repo *pRepo, const char* pszRepoName, - PTDNF_REPO_METADATA pRepoMD + PTDNF_REPO_METADATA pRepoMD, + char * pszSnapshotTime ); uint32_t @@ -736,6 +737,12 @@ TDNFAlterRepoState( const char* pszId ); +uint32_t +TDNFExcludeFromSnapshot( + PTDNF_REPO_DATA pRepos, + const char* pszId + ); + uint32_t TDNFCloneRepo( PTDNF_REPO_DATA pRepoIn, diff --git a/client/repo.c b/client/repo.c index bf4db830..eb668517 100644 --- a/client/repo.c +++ b/client/repo.c @@ -24,6 +24,8 @@ TDNFInitRepo( Pool* pPool = NULL; int nUseMetaDataCache = 0; PSOLV_REPO_INFO_INTERNAL pSolvRepoInfo = NULL; + PTDNF_CMD_OPT pSetOpt = NULL; + char * pszSnapshotTime = NULL; if (!pTdnf || !pRepoData || !pSack || !pSack->pPool) { @@ -31,6 +33,21 @@ TDNFInitRepo( BAIL_ON_TDNF_ERROR(dwError); } + // set local POSIX limit if conf or cmd line opt is present + if (pTdnf->pConf != NULL && pTdnf->pConf->pszSnapshotTime!= NULL) + { + pszSnapshotTime = pTdnf->pConf->pszSnapshotTime; + } + + // take command line over config if both are present + for (pSetOpt = pTdnf->pArgs->pSetOpt; pSetOpt; pSetOpt = pSetOpt->pNext) + { + if(strncmp(pSetOpt->pszOptName, TDNF_CONF_KEY_SNAPSHOT_TIME, strlen(TDNF_CONF_KEY_SNAPSHOT_TIME)) == 0) + { + pszSnapshotTime = pSetOpt->pszOptValue; + } + } + pPool = pSack->pPool; dwError = TDNFGetCachePath(pTdnf, pRepoData, @@ -82,20 +99,27 @@ TDNFInitRepo( pRepo->appdata = pSolvRepoInfo; if (pRepoData->nHasMetaData) { - dwError = SolvCalculateCookieForFile(pRepoMD->pszRepoMD, pSolvRepoInfo->cookie); - BAIL_ON_TDNF_ERROR(dwError); - pSolvRepoInfo->nCookieSet = 1; - - dwError = SolvUseMetaDataCache(pSack, pSolvRepoInfo, &nUseMetaDataCache); - BAIL_ON_TDNF_ERROR(dwError); - - if (nUseMetaDataCache == 0) { - dwError = TDNFInitRepoFromMetadata(pRepo, pRepoData->pszId, pRepoMD); + if (!pRepoData->nExcludeSnapshot && pszSnapshotTime != NULL) { + dwError = TDNFInitRepoFromMetadata(pRepo, pRepoData->pszId, pRepoMD, pszSnapshotTime); BAIL_ON_TDNF_ERROR(dwError); + } else { + dwError = SolvCalculateCookieForFile(pRepoMD->pszRepoMD, pSolvRepoInfo->cookie); + BAIL_ON_TDNF_ERROR(dwError); + pSolvRepoInfo->nCookieSet = 1; - dwError = SolvCreateMetaDataCache(pSack, pSolvRepoInfo); + dwError = SolvUseMetaDataCache(pSack, pSolvRepoInfo, &nUseMetaDataCache); BAIL_ON_TDNF_ERROR(dwError); + + //force load from repo if POSIX time limit is present + if (nUseMetaDataCache == 0) { + dwError = TDNFInitRepoFromMetadata(pRepo, pRepoData->pszId, pRepoMD, NULL); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvCreateMetaDataCache(pSack, pSolvRepoInfo); + BAIL_ON_TDNF_ERROR(dwError); + } } + } else { dwError = SolvReadRpmsFromDirectory(pRepo, pRepoData->ppszBaseUrls[0]); BAIL_ON_TDNF_ERROR(dwError); @@ -136,7 +160,8 @@ uint32_t TDNFInitRepoFromMetadata( Repo *pRepo, const char* pszRepoName, - PTDNF_REPO_METADATA pRepoMD + PTDNF_REPO_METADATA pRepoMD, + char * pszSnapshotTime ) { uint32_t dwError = 0; @@ -153,7 +178,8 @@ TDNFInitRepoFromMetadata( pRepoMD->pszPrimary, pRepoMD->pszFileLists, pRepoMD->pszUpdateInfo, - pRepoMD->pszOther); + pRepoMD->pszOther, + pszSnapshotTime); cleanup: return dwError; diff --git a/client/repolist.c b/client/repolist.c index bbf9d7d0..244765ff 100644 --- a/client/repolist.c +++ b/client/repolist.c @@ -380,6 +380,7 @@ TDNFCreateRepo( BAIL_ON_TDNF_ERROR(dwError); pRepo->nEnabled = TDNF_REPO_DEFAULT_ENABLED; + pRepo->nExcludeSnapshot = 0; pRepo->nHasMetaData = 1; pRepo->nSkipIfUnavailable = TDNF_REPO_DEFAULT_SKIP; pRepo->nGPGCheck = TDNF_REPO_DEFAULT_GPGCHECK; @@ -675,6 +676,7 @@ TDNFRepoListFinalize( PTDNF_CMD_OPT pSetOpt = NULL; PTDNF_REPO_DATA pRepo = NULL; int nRepoidSeen = 0; + char ** ppszRepos = NULL; if(!pTdnf || !pTdnf->pArgs || !pTdnf->pRepos) { @@ -715,6 +717,22 @@ TDNFRepoListFinalize( 1, pSetOpt->pszOptValue); } + else if (strcmp(pSetOpt->pszOptName, "snapshotexcluderepos") == 0) + { + ppszRepos = NULL; + int i = 0; + dwError = TDNFSplitStringToArray(pSetOpt->pszOptValue, ",", &ppszRepos); + BAIL_ON_TDNF_ERROR(dwError); + + while (ppszRepos && ppszRepos[i]){ + dwError = TDNFExcludeFromSnapshot( + pTdnf->pRepos, + ppszRepos[i]); + BAIL_ON_TDNF_ERROR(dwError); + i++; + } + + } BAIL_ON_TDNF_ERROR(dwError); } @@ -775,6 +793,7 @@ TDNFRepoListFinalize( BAIL_ON_TDNF_ERROR(dwError); } cleanup: + TDNFFreeStringArray(ppszRepos); return dwError; error: goto cleanup; @@ -826,6 +845,53 @@ TDNFAlterRepoState( goto cleanup; } +uint32_t +TDNFExcludeFromSnapshot( + PTDNF_REPO_DATA pRepos, + const char* pszId + ) +{ + uint32_t dwError = 0; + int nIsGlob = 0; + if(!pRepos || IsNullOrEmptyString(pszId)) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + nIsGlob = TDNFIsGlob(pszId); + + for (int nMatch = 0; pRepos; pRepos = pRepos->pNext) + { + if(nIsGlob) + { + if(!fnmatch(pszId, pRepos->pszId, 0)) + { + nMatch = 1; + } + } + else if(!strcmp(pRepos->pszId, pszId)) + { + nMatch = 1; + } + + if(nMatch) + { + pRepos->nExcludeSnapshot = 1; + if(!nIsGlob) + { + break; + } + } + } + +cleanup: + return dwError; + +error: + goto cleanup; +} + uint32_t TDNFCloneRepo( PTDNF_REPO_DATA pRepoIn, diff --git a/common/config.h b/common/config.h index f8a82640..3c657024 100644 --- a/common/config.h +++ b/common/config.h @@ -23,6 +23,7 @@ #define TDNF_CONF_KEY_GPGCHECK "gpgcheck" #define TDNF_CONF_KEY_INSTALLONLY_LIMIT "installonly_limit" #define TDNF_CONF_KEY_INSTALLONLYPKGS "installonlypkgs" +#define TDNF_CONF_KEY_SNAPSHOT_TIME "snapshottime" #define TDNF_CONF_KEY_CLEAN_REQ_ON_REMOVE "clean_requirements_on_remove" #define TDNF_CONF_KEY_REPODIR "repodir" // typo, keep for back compatibility #define TDNF_CONF_KEY_REPOSDIR "reposdir" diff --git a/etc/bash_completion.d/tdnf-completion.bash b/etc/bash_completion.d/tdnf-completion.bash index 4e480404..46a62016 100644 --- a/etc/bash_completion.d/tdnf-completion.bash +++ b/etc/bash_completion.d/tdnf-completion.bash @@ -26,6 +26,11 @@ _tdnf__process_if_prev_is_option() COMPREPLY=( $(compgen -W "$opts" -- $cur) ) return 0 ;; + --snapshotexcluderepos) + opts=`tdnf repolist enabled | awk '{if (NR > 1) print $1}'` + COMPREPLY=( $(compgen -W "$opts" -- $cur) ) + return 0 + ;; --installroot) COMPREPLY=( $(compgen -d -- $cur) ) return 0 @@ -92,7 +97,7 @@ _tdnf() { local c=0 cur __opts __cmds COMPREPLY=() - __opts="--assumeno --assumeyes --cacheonly --debugsolver --disableexcludes --disableplugin --disablerepo --downloaddir --downloadonly --enablerepo --enableplugin --exclude --installroot --noautoremove --nogpgcheck --noplugins --quiet --reboot --refresh --releasever --repo --repofrompath --repoid --rpmverbosity --security --sec --setopt --skip --skipconflicts --skipdigest --skipsignature --skipobsoletes --testonly --version --available --duplicates --extras --file --installed --whatdepends --whatrequires --whatenhances --whatobsoletes --whatprovides --whatrecommends --whatrequires --whatsuggests --whatsupplements --depends --enhances --list --obsoletes --provides --recommends --requires --requires --suggests --source --supplements --arch --delete --download --download --gpgcheck --metadata --newest --norepopath --source --urls" + __opts="--assumeno --assumeyes --cacheonly --debugsolver --disableexcludes --disableplugin --disablerepo --downloaddir --downloadonly --enablerepo --enableplugin --exclude --installroot --noautoremove --nogpgcheck --noplugins --quiet --reboot --refresh --releasever --repo --repofrompath --repoid --rpmverbosity --security --sec --setopt --skip --skipconflicts --skipdigest --skipsignature --skipobsoletes --snapshotexcluderepos --snapshottime --testonly --version --available --duplicates --extras --file --installed --whatdepends --whatrequires --whatenhances --whatobsoletes --whatprovides --whatrecommends --whatrequires --whatsuggests --whatsupplements --depends --enhances --list --obsoletes --provides --recommends --requires --requires --suggests --source --supplements --arch --delete --download --download --gpgcheck --metadata --newest --norepopath --source --urls" __cmds="autoerase autoremove check check-local check-update clean distro-sync downgrade erase help history info install list makecache mark provides whatprovides reinstall remove repolist repoquery reposync search update update-to updateinfo upgrade upgrade-to" cur="${COMP_WORDS[COMP_CWORD]}" _tdnf__process_if_prev_is_option && return 0 diff --git a/include/tdnferror.h b/include/tdnferror.h index a53058d3..53d38f77 100644 --- a/include/tdnferror.h +++ b/include/tdnferror.h @@ -187,6 +187,15 @@ extern "C" { #define ERROR_TDNF_HISTORY_ERROR 1801 #define ERROR_TDNF_HISTORY_NODB 1802 +#define ERROR_TDNF_TIME_FILTER_BASE 1900 +// filter MEMORY +#define ERROR_TDNF_TIME_FILTER_MEMORY (ERROR_TDNF_TIME_FILTER_BASE + 1) +// filter parsing error +#define ERROR_TDNF_TIME_FILTER_PARSE (ERROR_TDNF_TIME_FILTER_BASE + 2) +// filter IO error +#define ERROR_TDNF_TIME_FILTER_IO (ERROR_TDNF_TIME_FILTER_BASE + 3) +// filter general error +# define ERROR_TDNF_TIME_FILTER_GENERAL (ERROR_TDNF_TIME_FILTER_BASE + 4) #define ERROR_TDNF_PLUGIN_BASE 2000 diff --git a/include/tdnftypes.h b/include/tdnftypes.h index f07da8e6..e424706d 100644 --- a/include/tdnftypes.h +++ b/include/tdnftypes.h @@ -9,6 +9,7 @@ #pragma once #include +#include #ifdef __cplusplus extern "C" { @@ -259,6 +260,7 @@ typedef struct _TDNF_CONF int nDistroSyncReinstallChanged; int nPluginsEnabled; char* pszRepoDir; + char* pszSnapshotTime; char* pszCacheDir; char* pszPersistDir; char* pszProxy; @@ -283,6 +285,7 @@ typedef struct _TDNF_CONF typedef struct _TDNF_REPO_DATA { int nEnabled; + int nExcludeSnapshot; int nSkipIfUnavailable; int nGPGCheck; int nHasMetaData; diff --git a/solv/defines.h b/solv/defines.h index 38f5ab1d..ddb33554 100644 --- a/solv/defines.h +++ b/solv/defines.h @@ -18,4 +18,33 @@ } \ } while(0) +typedef struct { + // frequently changed values + char * pszElementBuffer; + int nBufferLen; + int nInPackage; + int nPrintPackage; + int nTimeFound; + + // managed values + int nBufferMaxLen; + int nDepth; + int nPrevElement; // enum 0 -> start, 1 -> data, 2 -> end + + //set and forget on creation + time_t nSearchTime; + FILE * pbOutfile; +} TDNFFilterData; + +#define TDNF_MAX_FILTER_INPUT_THRESHOLD 500000000 +#define TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE 16000 + +#define BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError) \ + do { \ + if (dwError) \ + { \ + goto error; \ + } \ + } while(0) + #endif /* __SOLV_DEFINES_H__ */ diff --git a/solv/includes.h b/solv/includes.h index 2ab0c5c8..1e39ecbb 100644 --- a/solv/includes.h +++ b/solv/includes.h @@ -10,7 +10,6 @@ #include #include #include -#include #include // libsolv @@ -44,4 +43,9 @@ #include "../history/history.h" #include "prototypes.h" +#include +#include +#include +#include + #endif /* __SOLV_INCLUDES_H__ */ diff --git a/solv/prototypes.h b/solv/prototypes.h index 68c30e60..c0e8612d 100644 --- a/solv/prototypes.h +++ b/solv/prototypes.h @@ -520,7 +520,8 @@ SolvReadYumRepo( const char *pszPrimary, const char *pszFilelists, const char *pszUpdateinfo, - const char *pszOther + const char *pszOther, + const char *pszSnapshotTime ); uint32_t diff --git a/solv/tdnfrepo.c b/solv/tdnfrepo.c index 586059b9..9f620497 100644 --- a/solv/tdnfrepo.c +++ b/solv/tdnfrepo.c @@ -9,6 +9,766 @@ #include "includes.h" +// #### XML FILTER CODE #### + +/*** +* Resize the buffer specified by ppszCharBuffer and update pnBufferMaxLen +* to the length of the newly resized buffer if the nLengthToAdd would overflow +* the buffer. +***/ +uint32_t +checkAndResizeBuffer(char ** ppszCharBuffer, int * pnBufferMaxLen, int nLengthToAdd) { + uint32_t dwError = 0; + char * pszTempCharBuffer = NULL; + if (ppszCharBuffer == NULL || *ppszCharBuffer == NULL || pnBufferMaxLen == NULL || *pnBufferMaxLen <= 0 || nLengthToAdd < 0) { + dwError = ERROR_TDNF_TIME_FILTER_GENERAL; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + // calculate new max length + int nTempMaxLen = *pnBufferMaxLen; + int nBufferContentLen = strlen(*ppszCharBuffer); + while (nBufferContentLen + nLengthToAdd + 1 >= nTempMaxLen) + { + nTempMaxLen *= 2; + } + if (nTempMaxLen >= TDNF_MAX_FILTER_INPUT_THRESHOLD) + { + dwError = ERROR_TDNF_TIME_FILTER_GENERAL; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + // only realloc if the size changed + if (nTempMaxLen != *pnBufferMaxLen) + { + pszTempCharBuffer = realloc(*ppszCharBuffer, nTempMaxLen); + if (!pszTempCharBuffer) + { + dwError = ERROR_TDNF_TIME_FILTER_MEMORY; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + //set expanded char buffer + *ppszCharBuffer = pszTempCharBuffer; + *pnBufferMaxLen = nTempMaxLen; + } + +cleanup: + return dwError; +error: + pr_err("An error occurred during buffer resizing with the following code: %u\n", dwError); + goto cleanup; +} + +/*** +* allocate a new string in ppszDestStr location with the linted description, +* all '&', '<', and '>' characters will be replaced with the xml escape +* character versions of each in line. +***/ +uint32_t +xmlEscapeCharLinter(const char * pszStringToEscape, char ** ppszDestStr) { + uint32_t dwError = 0; + const char * amp = "&"; + const char * gt = ">"; + const char * lt = "<"; + + if (pszStringToEscape == NULL || ppszDestStr == NULL) + { + dwError = ERROR_TDNF_TIME_FILTER_GENERAL; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + // allocate new string for linted string + int nStrToLintLen = (strlen(pszStringToEscape) + 1); // add one for null char + char * pszLintedStr = malloc(nStrToLintLen * sizeof(char)); + if (!pszLintedStr) + { + dwError = ERROR_TDNF_TIME_FILTER_MEMORY; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + bzero(pszLintedStr, nStrToLintLen * sizeof(char)); + int nOffset = 0; + int nLintedSize = nStrToLintLen; + + // Loop through string to lint looking for chars in need of escaping + for (int i = 0; i < nStrToLintLen; i++) + { + char * pszCharToAdd = NULL; + int nAddStrlen = 1; + // check current char for escape character + switch (pszStringToEscape[i]) + { + case '&': + pszCharToAdd = amp; + break; + case '>': + pszCharToAdd = gt; + break; + case '<': + pszCharToAdd = lt; + break; + } + + //resize buffer if needed + if (pszCharToAdd != NULL) + { + nAddStrlen = strlen(pszCharToAdd); + } + dwError = checkAndResizeBuffer(&pszLintedStr, &nLintedSize, nAddStrlen); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + // add linted char + if (pszCharToAdd == NULL) + { + pszLintedStr[i + nOffset] = pszStringToEscape[i]; + } + else + { + strcat(pszLintedStr, pszCharToAdd); + nOffset += nAddStrlen - 1; // minus 1 to account for the original space used by the character + } + } + + // set Dest to linted string if all done + *ppszDestStr = pszLintedStr; + +cleanup: + return dwError; +error: + pr_err("An error occurred during escape character linting with the following code: %u\n", dwError); + goto cleanup; +} + +/*** +* allocate a new buffer to location pszElementBuffer of the size +* nElementBufferMax or greater (in the case resizing is needed). +* a formatted start element with the name and attrs specified will be +* placed in the newly allocated buffer. +***/ +uint32_t +addElementStartToBuffer(char ** pszElementBuffer, int * nElementBufferMax, const char * pszElementName, const char ** ppszAttrs) { + uint32_t dwError = 0; + + if (pszElementBuffer == NULL || nElementBufferMax == NULL || *nElementBufferMax < 0) + { + dwError = ERROR_TDNF_TIME_FILTER_GENERAL; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + // set default buffer max length + if (*nElementBufferMax == 0) + { + *nElementBufferMax = TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE; + } + *pszElementBuffer = malloc(*nElementBufferMax * sizeof(char)); + + char * pszLintedAttrVal = NULL; + char * pszTempBuffer = NULL; + dwError = checkAndResizeBuffer(pszElementBuffer, nElementBufferMax, strlen(pszElementName) + 2); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + sprintf(*pszElementBuffer, "<%s", pszElementName); + for (int i = 0; ppszAttrs[i]; i += 2) + { + dwError = xmlEscapeCharLinter(ppszAttrs[i+1], &pszLintedAttrVal); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + int nTempBufferLen = strlen(pszLintedAttrVal) + strlen(ppszAttrs[i]) + 5; + dwError = checkAndResizeBuffer(pszElementBuffer, nElementBufferMax, nTempBufferLen); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + pszTempBuffer = malloc(sizeof(char) * nTempBufferLen); + if (!pszTempBuffer) + { + dwError = ERROR_TDNF_TIME_FILTER_MEMORY; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + sprintf(pszTempBuffer, " %s=\"%s\"", ppszAttrs[i], pszLintedAttrVal); + strcat(*pszElementBuffer, pszTempBuffer); + + // free temp variables + free(pszTempBuffer); + pszTempBuffer = NULL; + free(pszLintedAttrVal); + pszLintedAttrVal = NULL; + } + strcat(*pszElementBuffer, ">"); + +cleanup: + if (pszLintedAttrVal) + { + free(pszLintedAttrVal); + } + if (pszTempBuffer) + { + free(pszTempBuffer); + } + return dwError; +error: + pr_err("An error occurred during start element generation with the following code: %u\n", dwError); + goto cleanup; +} + +/*** + * + ***/ +uint32_t +addElementEndToBuffer(char ** pszElementBuffer, int * nElementBufferMaxLen, const char * pszElementName) { + uint32_t dwError = 0; + if (pszElementBuffer == NULL || nElementBufferMaxLen == NULL || *nElementBufferMaxLen < 0) + { + dwError = ERROR_TDNF_TIME_FILTER_GENERAL; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + if (*nElementBufferMaxLen == 0 ) + { + *nElementBufferMaxLen = TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE; + } + *pszElementBuffer = malloc(*nElementBufferMaxLen * sizeof(char)); + + dwError = checkAndResizeBuffer(pszElementBuffer, nElementBufferMaxLen, strlen(pszElementName) + 4); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + sprintf(*pszElementBuffer, "", pszElementName); + +cleanup: + return dwError; +error: + pr_err("An error occurred during end element generation with the following code: %u\n", dwError); + goto cleanup; +} + +/*** + * + ***/ +uint32_t +printElementStartToFile(FILE * pbOutfile, const char * pszElementName, const char ** ppszAttrs) { + uint32_t dwError = 0; + if (pbOutfile == NULL) + { + dwError = ERROR_TDNF_TIME_FILTER_GENERAL; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + int nStartElementBufferLength = TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE; + char * pszStartElement = NULL; + + dwError = addElementStartToBuffer(&pszStartElement, &nStartElementBufferLength, pszElementName, ppszAttrs); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + fprintf(pbOutfile, "%s", pszStartElement); + if (ferror(pbOutfile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + +cleanup: + if (pszStartElement) + { + free(pszStartElement); + } + return dwError; +error: + pr_err("An error occurred during start element printing with the following code: %u\n", dwError); + goto cleanup; +} + +/*** + * + ***/ +uint32_t +printElementEndToFile(FILE * pbOutfile, const char * pszElementName) { + uint32_t dwError = 0; + if (pbOutfile == NULL) + { + dwError = ERROR_TDNF_TIME_FILTER_GENERAL; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + int nEndElementBufferLength = TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE; + char * pszEndElement = NULL; + + dwError = addElementEndToBuffer(&pszEndElement, &nEndElementBufferLength, pszElementName); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + fprintf(pbOutfile, "%s", pszEndElement); + if (ferror(pbOutfile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + +cleanup: + if (pszEndElement) + { + free(pszEndElement); + } + return dwError; +error: + pr_err("An error occurred during end element printing with the following code: %u\n", dwError); + goto cleanup; +} + +/*** + * + ***/ +void +TDNFFilterStartElement(void *userData, const char * name, const char ** attrs) { + uint32_t dwError = 0; + char * pszStartElementBuffer = NULL; + // load tracking data + TDNFFilterData * pTracking = (TDNFFilterData *)userData; + int nAddNewLineAfterStart = pTracking->nPrevElement == 0; + char szNewLineBuffer[2]; + if (nAddNewLineAfterStart) + { + sprintf(szNewLineBuffer, "\n"); + } + else + { + bzero(szNewLineBuffer, sizeof(szNewLineBuffer)); // don't assume memory zero'd + } + + // increment depth + pTracking->nDepth += 1; + pTracking->nPrevElement = 0; + + // new package to parse or currently parsing package info + if (strcmp(name, "package") == 0 || pTracking->nInPackage) + { + pTracking->nInPackage = 1; + + // already found/checked time + if (pTracking->nTimeFound && pTracking->nPrintPackage) + { + fprintf(pTracking->pbOutfile, "%s", szNewLineBuffer); + if (ferror(pTracking->pbOutfile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + dwError = printElementStartToFile(pTracking->pbOutfile, name, attrs); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + } + else + { // still checking for time + if (strcmp(name, "time") == 0) + { + // time found + // validate file POSIX time + for (int i = 0; attrs[i]; i += 2) + { + if (strcmp(attrs[i], "file") == 0) + { + // file time is the time the package is published to the repo + // when this is less than our search time, allow the package to be + // printed to the temp repo file, otherwise the current package + // can be discarded. + errno = 0; + char * pszSnapshotTimeEnd = NULL; + long nCurrentPackageTime = strtoll(attrs[i+1], pszSnapshotTimeEnd, 10); + if (errno || pszSnapshotTimeEnd == attrs[i+1]) + { + dwError = ERROR_TDNF_TIME_FILTER_PARSE; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + pTracking->nPrintPackage = (nCurrentPackageTime <= pTracking->nSearchTime); + pTracking->nTimeFound = 1; + break; + } + } + if (pTracking->nPrintPackage) + { + // print buffer when time is found + fprintf(pTracking->pbOutfile, "%s", pTracking->pszElementBuffer); + if (ferror(pTracking->pbOutfile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + fprintf(pTracking->pbOutfile, "%s", szNewLineBuffer); + if (ferror(pTracking->pbOutfile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + // print time element + dwError = printElementStartToFile(pTracking->pbOutfile, name, attrs); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + } + } + else if (!pTracking->nTimeFound) + { + // if we haven't found a time yet, the element must be stored + // add to file buffer + int nStartElementBufferSize = TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE; + pszStartElementBuffer = NULL; + + dwError = addElementStartToBuffer(&pszStartElementBuffer, &nStartElementBufferSize, name, attrs); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + int nLenToAdd = strlen(pszStartElementBuffer); + nLenToAdd += strlen(szNewLineBuffer); // +1 if newLine character present + + dwError = checkAndResizeBuffer(&pszStartElementBuffer, &nStartElementBufferSize, strlen(szNewLineBuffer)); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + strcat(pszStartElementBuffer, szNewLineBuffer); + + dwError = checkAndResizeBuffer(&(pTracking->pszElementBuffer), &(pTracking->nBufferMaxLen), nLenToAdd); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + strcat(pTracking->pszElementBuffer, pszStartElementBuffer); + } + } + } + else + { // not in a package or parsing a new package + fprintf(pTracking->pbOutfile, "%s", szNewLineBuffer); + if (ferror(pTracking->pbOutfile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + // output line + dwError = printElementStartToFile(pTracking->pbOutfile, name, attrs); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + } +cleanup: + if (pszStartElementBuffer) + { + free(pszStartElementBuffer); + } + return; +error: + pr_err("An error occurred during start element parsing with the following code: %u\n", dwError); + goto cleanup; +} + +/*** + * + ***/ +void +TDNFFilterEndElement(void * userData, const char * name) { + uint32_t dwError = 0; + char * pszElementBuffer = NULL; + // load tracking data + TDNFFilterData * pTracking = (TDNFFilterData *)userData; + + // decrement depth + pTracking->nDepth -= 1; + pTracking->nPrevElement = 2; + + if (!pTracking->nInPackage || pTracking->nPrintPackage) + { + // print end element to file + dwError = printElementEndToFile(pTracking->pbOutfile, name); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + } + else if (pTracking->nInPackage && !pTracking->nTimeFound) + { + int nEndElementBufferLen = TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE; + pszElementBuffer = NULL; + + // add end element to buffer + dwError = addElementEndToBuffer(&pszElementBuffer, &nEndElementBufferLen, name); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + int nEndElementLen = strlen(pszElementBuffer); + + dwError = checkAndResizeBuffer(&(pTracking->pszElementBuffer), &(pTracking->nBufferMaxLen), nEndElementLen); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + strcat(pTracking->pszElementBuffer, pszElementBuffer); + + } // else do nothing + + if (strcmp(name, "package") == 0) + { // on end package, reset tracking function + // reset userData + pTracking->nBufferLen = 0; + bzero(pTracking->pszElementBuffer, pTracking->nBufferMaxLen); + pTracking->nInPackage = 0; + pTracking->nPrintPackage = 0; + pTracking->nTimeFound = 0; + } +cleanup: + if (pszElementBuffer) + { + free(pszElementBuffer); + } + return; +error: + pr_err("An error occurred during end element parsing with the following code: %u\n", dwError); + goto cleanup; +} + +/*** + * + ***/ +void +TDNFFilterCharDataHandler(void * userData, const char * content, int length) { + uint32_t dwError = 0; + // load tracking data + TDNFFilterData * pTracking = (TDNFFilterData *)userData; + pTracking->nPrevElement = 1; + + char * pszCharData = malloc((length + 1) * sizeof(char)); + if (!pszCharData) + { + dwError = ERROR_TDNF_TIME_FILTER_MEMORY; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + bzero(pszCharData, (length + 1) * sizeof(char)); + strncpy(pszCharData, content, length); + char * pszLintedCharData = NULL; + dwError = xmlEscapeCharLinter(pszCharData, &pszLintedCharData); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + // check params + if (!pTracking->nInPackage || pTracking->nPrintPackage) + { + // print to file + fprintf(pTracking->pbOutfile, "%s", pszLintedCharData); + if (ferror(pTracking->pbOutfile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + } + else if (pTracking->nInPackage && !pTracking->nTimeFound) + { + // add to buffer + dwError = checkAndResizeBuffer(&(pTracking->pszElementBuffer), &(pTracking->nBufferMaxLen), strlen(pszLintedCharData)); + if (dwError) + { + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + strcat(pTracking->pszElementBuffer, pszLintedCharData); + } // else do nothing (skipped package) + +cleanup: + if (pszLintedCharData) + { + free(pszLintedCharData); + } + if (pszCharData) + { + free(pszCharData); + } + return; +error: + pr_err("An error occurred during char data handling with the following code: %u\n", dwError); + goto cleanup; +} + +/*** + * + ***/ +char * +TDNFFilterFile(const char * pszInFilePath, const char * pszSnapshotTime) { + // vars + uint32_t dwError = 0; + TDNFFilterData pData; + bzero(&pData, sizeof(TDNFFilterData)); + time_t nSnapshotTime; + bzero(&nSnapshotTime, sizeof(time_t)); + XML_Parser bParser; + bzero(&bParser, sizeof(XML_Parser)); + FILE * pbInFile = NULL; + FILE * pbOutFile = NULL; + char pszTimeExtension[100]; + char * pszOutFilePath = NULL; + + // convert snapshot string to time for use by the parser and the temp file name + errno = 0; + char * pszSnapshotTimeEnd = NULL; + nSnapshotTime = strtoll(pszSnapshotTime, &pszSnapshotTimeEnd, 10); + if (errno || pszSnapshotTimeEnd == pszSnapshotTime) + { + dwError = ERROR_TDNF_TIME_FILTER_PARSE; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + //create output file ending + sprintf(pszTimeExtension, "-%lld.xml", nSnapshotTime); + + // find total extension length + int nInFileExtLen = 4; // len of ".xml" + char * pszFileExt = strrchr(pszInFilePath, '.'); + if (strcmp(pszFileExt, ".xml") != 0) + { + nInFileExtLen += strlen(pszFileExt); + } + + // calculate outfile length and allocate + int nInFileLen = strlen(pszInFilePath); + int nOutFileLen = (nInFileLen - nInFileExtLen) + strlen(pszTimeExtension) + 1; + pszOutFilePath = malloc(nOutFileLen * sizeof(char)); + if (!pszOutFilePath) + { + dwError = ERROR_TDNF_TIME_FILTER_MEMORY; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + bzero(pszOutFilePath, nOutFileLen * sizeof(char)); + + // use infile path + timestamp as new output file + strncpy(pszOutFilePath, pszInFilePath, nInFileLen - nInFileExtLen); // remove extension to be added with the name + strcat(pszOutFilePath, pszTimeExtension); + + // init vars, load files + pbInFile = solv_xfopen(pszInFilePath, "r"); + if (!pbInFile) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + pbOutFile = fopen(pszOutFilePath, "w"); + if (!pbOutFile) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + pData.nBufferMaxLen = TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE; + pData.pszElementBuffer = (char *)malloc(pData.nBufferMaxLen * sizeof(char)); + if (!pData.pszElementBuffer) + { + dwError = ERROR_TDNF_TIME_FILTER_MEMORY; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + bzero(pData.pszElementBuffer, pData.nBufferMaxLen); + pData.pbOutfile = pbOutFile; + pData.nSearchTime = nSnapshotTime; + pData.nDepth = 0; + pData.nBufferLen = 0; + pData.nInPackage = 0; + pData.nPrintPackage = 0; + pData.nTimeFound = 0; + + //create parser + bParser = XML_ParserCreate(NULL); + if (!bParser) + { + dwError = ERROR_TDNF_TIME_FILTER_PARSE; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + XML_SetUserData(bParser, &pData); + XML_SetElementHandler(bParser, TDNFFilterStartElement, TDNFFilterEndElement); + XML_SetCharacterDataHandler(bParser, TDNFFilterCharDataHandler); + + //parse XML + fprintf(pbOutFile, "\n"); + if (ferror(pbOutFile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + int nInputEof; + do + { + void * pszXMLParseBuffer = XML_GetBuffer(bParser, BUFSIZ); + if (!pszXMLParseBuffer) + { + fprintf(stderr, "Couldn't allocate memory for buffer\n"); + dwError = ERROR_TDNF_TIME_FILTER_MEMORY; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + const size_t len = fread(pszXMLParseBuffer, 1, BUFSIZ - 1, pbInFile); + ((char *)pszXMLParseBuffer)[len] = '\0'; + if (ferror(pbInFile)) + { + dwError = ERROR_TDNF_TIME_FILTER_IO; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + + nInputEof = feof(pbInFile); + + if (XML_ParseBuffer(bParser, (int)len, nInputEof) == XML_STATUS_ERROR) + { + fprintf(stderr, + "Parse error at line %lu:\n%s\n", + XML_GetCurrentLineNumber(bParser), + XML_ErrorString(XML_GetErrorCode(bParser))); + dwError = ERROR_TDNF_TIME_FILTER_PARSE; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + } while (!nInputEof); + +cleanup: + if (pData.pszElementBuffer) { + free(pData.pszElementBuffer); + } + + if (bParser) + { + XML_ParserFree(bParser); + } + + if (pbOutFile) + { + fclose(pbOutFile); + } + + if (pbInFile) + { + fclose(pbInFile); + } + + return pszOutFilePath; +error: + pr_err("An error occurred during snapshot filtering with the following code: %u\n", dwError); + goto cleanup; +} +// #### END XML SNAPSHOT FILTER CODE #### + uint32_t SolvLoadRepomd( Repo* pRepo, @@ -195,10 +955,14 @@ SolvReadYumRepo( const char *pszPrimary, const char *pszFilelists, const char *pszUpdateinfo, - const char *pszOther + const char *pszOther, + const char *pszSnapshotTime ) { uint32_t dwError = 0; + // new vars for Filter + char * tempPrimaryRepoFile = NULL; + // end new vars if(!pRepo || !pszRepoName || !pszRepomd || !pszPrimary) { dwError = ERROR_TDNF_INVALID_PARAMETER; @@ -209,6 +973,18 @@ SolvReadYumRepo( BAIL_ON_TDNF_LIBSOLV_ERROR(dwError); + // Run filter if option present + if (pszSnapshotTime != NULL){ + tempPrimaryRepoFile = TDNFFilterFile(pszPrimary, pszSnapshotTime); + if (tempPrimaryRepoFile == NULL) + { + dwError = ERROR_TDNF_TIME_FILTER_GENERAL; + BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError); + } + pszPrimary = tempPrimaryRepoFile; + } + // End filter code + dwError = SolvLoadRepomdPrimary(pRepo, pszPrimary); BAIL_ON_TDNF_LIBSOLV_ERROR(dwError); @@ -232,6 +1008,11 @@ SolvReadYumRepo( cleanup: + if(tempPrimaryRepoFile != NULL) + { + remove(tempPrimaryRepoFile); + free(tempPrimaryRepoFile); + } return dwError; diff --git a/tools/cli/lib/help.c b/tools/cli/lib/help.c index bc4cf83e..6c1d3499 100644 --- a/tools/cli/lib/help.c +++ b/tools/cli/lib/help.c @@ -44,6 +44,8 @@ static const char *help_msg = " [--skipdigest]\n" " [--skipsignature]\n" " [--skipobsoletes]\n" + " [--snapshotexcluderepos=[,,...]\n" + " [--snapshottime=]\n" " [--testonly]\n" " [--version]\n\n" "repoquery select options:\n" diff --git a/tools/cli/lib/parseargs.c b/tools/cli/lib/parseargs.c index deedb1bd..d2417d24 100644 --- a/tools/cli/lib/parseargs.c +++ b/tools/cli/lib/parseargs.c @@ -57,6 +57,8 @@ static struct option pstOptions[] = {"skipdigest", no_argument, 0, 0}, //--skipdigest to skip verifying RPM digest {"skipobsoletes", no_argument, 0, 0}, //--skipobsoletes to skip obsolete problems {"skipsignature", no_argument, 0, 0}, //--skipsignature to skip verifying RPM signatures + {"snapshotexcluderepos", required_argument, 0, 0}, //--snapshotexcluderepos + {"snapshottime", required_argument, 0, 0}, //--snapshottime {"source", no_argument, &_opt.nSource, 1}, {"testonly", no_argument, &_opt.nTestOnly, 1}, {"verbose", no_argument, &_opt.nVerbose, 1}, //-v --verbose