Skip to content

Commit c727983

Browse files
KevinJWKevinJW
committed
Add support for environment variables in minimal configuration archives
If archiving a minimal config and environment variables are used, it must be possible for all the needed variables to be resolved and any files pointed to should also exist at the time of archiving. The values of the environment will be written into the saved configuration in the environment section of the configuration file. Ths configuration stored thus may be different to the initial config being passed for archiving. Needs tests adding and futher discussion in the TSC meetings. Signed-off-by: Kevin Wheatley <[email protected]>
1 parent 54121ce commit c727983

File tree

5 files changed

+99
-54
lines changed

5 files changed

+99
-54
lines changed

‎include/OpenColorIO/OpenColorIO.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1504,7 +1504,7 @@ class OCIOEXPORT Config
15041504
*
15051505
* \return bool Archivable if true.
15061506
*/
1507-
bool isArchivable() const;
1507+
bool isArchivable(bool minimal) const;
15081508

15091509
/**
15101510
* \brief Archive the config and its LUTs into the specified output stream.

‎src/OpenColorIO/Config.cpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5472,7 +5472,7 @@ ConfigIOProxyRcPtr Config::getConfigIOProxy() const
54725472
return getImpl()->m_context->getConfigIOProxy();
54735473
}
54745474

5475-
bool Config::isArchivable() const
5475+
bool Config::isArchivable(bool minimal) const
54765476
{
54775477
ConstContextRcPtr context = getCurrentContext();
54785478

@@ -5485,7 +5485,7 @@ bool Config::isArchivable() const
54855485
}
54865486

54875487
// Utility lambda to check the following criteria.
5488-
auto validatePathForArchiving = [](const std::string & path)
5488+
auto validatePathForArchiving = [&minimal](const std::string & path)
54895489
{
54905490
// Using the normalized path.
54915491
const std::string normPath = pystring::os::path::normpath(path);
@@ -5496,8 +5496,9 @@ bool Config::isArchivable() const
54965496
pystring::startswith(normPath, "..") ||
54975497
// 3) A context variable may not be located at the start of the path.
54985498
(ContainsContextVariables(path) && //TODO: if we can resolve context, do so and validate path to file
5499-
(StringUtils::Find(path, "$") == 0 ||
5500-
StringUtils::Find(path, "%") == 0)))
5499+
(!minimal &&
5500+
(StringUtils::Find(path, "$") == 0 ||
5501+
StringUtils::Find(path, "%") == 0))))
55015502
{
55025503
return false;
55035504
}

‎src/OpenColorIO/OCIOZArchive.cpp‎

Lines changed: 70 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include "Platform.h"
1717
#include "utils/StringUtils.h"
1818
#include "transforms/FileTransform.h"
19+
#include "Logging.h"
1920

2021
#include "OCIOZArchive.h"
2122

@@ -183,6 +184,9 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo
183184
if (!possibleFormats.empty())
184185
{
185186
// The extension is supported. Add the current file to the OCIOZ archive.
187+
std::ostringstream logMessage;
188+
logMessage << "Adding file: " << absPath;
189+
LogInfo(logMessage.str());
186190
if (mz_zip_writer_add_path(
187191
archiver, absPath.c_str(),
188192
configWorkingDirectory, 0, 1) != MZ_OK)
@@ -201,9 +205,9 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo
201205
}
202206
}
203207

204-
void addReferencedFiles(void * archiver, const Config & config)
208+
ContextRcPtr addReferencedFiles(void * archiver, const ConfigRcPtr & config)
205209
{
206-
ConstContextRcPtr context = config.getCurrentContext();
210+
ConstContextRcPtr context = config->getCurrentContext();
207211
ContextRcPtr ctxFilepath = Context::Create();
208212
ctxFilepath->setSearchPath(context->getSearchPath());
209213
ctxFilepath->setWorkingDir(context->getWorkingDir());
@@ -212,20 +216,42 @@ void addReferencedFiles(void * archiver, const Config & config)
212216
auto prefixLength = std::string(context->getWorkingDir()).length() + 1; // +1 add trailing '/' TODO: improve this
213217

214218
std::set<std::string> files;
215-
config.GetAllFileReferences(files);
219+
config->GetAllFileReferences(files);
220+
std::set<std::string> added_files;
216221
for (const auto &file : files)
217222
{
218223
const std::string resolvedPath = context->resolveFileLocation(file.c_str(), ctxFilepath);
219224
const std::string relativePath = resolvedPath.substr(prefixLength);
225+
std::ostringstream logMessage;
220226

221-
auto returnCode = mz_zip_writer_add_file(archiver, resolvedPath.c_str(), relativePath.c_str());
222-
if (returnCode != MZ_OK)
227+
if (added_files.find(relativePath) == added_files.end())
223228
{
224-
std::ostringstream os;
225-
os << "Could not write file " << resolvedPath << " to in-memory archive.()" << returnCode << ")";
226-
throw Exception(os.str().c_str());
229+
logMessage << "Adding file: " << file << " -> " << relativePath;
230+
LogInfo(logMessage.str());
231+
auto returnCode = mz_zip_writer_add_file(archiver, resolvedPath.c_str(), relativePath.c_str());
232+
if (returnCode != MZ_OK)
233+
{
234+
std::ostringstream os;
235+
os << "Could not write file " << resolvedPath << " to in-memory archive.()" << returnCode << ")";
236+
throw Exception(os.str().c_str());
237+
}
227238
}
239+
else
240+
{
241+
logMessage << "Skipping already added file: " << file << " -> " << relativePath;
242+
LogInfo(logMessage.str());
243+
}
244+
added_files.insert(relativePath);
228245
}
246+
247+
const auto variableCount = ctxFilepath->getNumStringVars();
248+
for (auto i = 0; i != variableCount; ++i)
249+
{
250+
std::ostringstream logMessage;
251+
logMessage << "Used Variable: " << ctxFilepath->getStringVarNameByIndex(i) << " -> " << ctxFilepath->getStringVarByIndex(i);
252+
LogInfo(logMessage.str());
253+
}
254+
return ctxFilepath;
229255
}
230256

231257
//////////////////////////////////////////////////////////////////////////////////////
@@ -250,8 +276,9 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
250276
mz_zip_file file_info;
251277

252278
flags = EnvironmentOverride(flags);
279+
const bool minimal = HasFlag(flags, ARCHIVE_FLAGS_MINIMAL);
253280

254-
if (!config.isArchivable()) // TODO: pass in flags?
281+
if (!config.isArchivable(minimal)) // TODO: pass in flags?
255282
{
256283
std::ostringstream os;
257284
os << "Config is not archivable.";
@@ -261,11 +288,6 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
261288
// Initialize.
262289
memset(&file_info, 0, sizeof(file_info));
263290

264-
// Retrieve and store the config as string.
265-
std::stringstream ss;
266-
config.serialize(ss);
267-
std::string configStr = ss.str();
268-
269291
// Write zip to memory stream.
270292
#if MZ_VERSION_BUILD >= 040000
271293
write_mem_stream = mz_stream_mem_create();
@@ -303,10 +325,38 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
303325
// Open the in-memory zip.
304326
if (mz_zip_writer_open(archiver, write_mem_stream, 0) == MZ_OK)
305327
{
328+
ConfigRcPtr editiableConfig = config.createEditableCopy(); // TODO: is this a little heavy handed just so we can modify the envronment?
329+
///////////////////////
330+
// Adding LUT files //
331+
///////////////////////
332+
if (minimal)
333+
{
334+
ContextRcPtr archivedContext = addReferencedFiles(archiver, editiableConfig);
335+
336+
// Need to make sure the used evironment context is in the config
337+
const auto variableCount = archivedContext->getNumStringVars();
338+
for (auto i = 0; i != variableCount; ++i)
339+
{
340+
editiableConfig->addEnvironmentVar(archivedContext->getStringVarNameByIndex(i), archivedContext->getStringVarByIndex(i));
341+
}
342+
}
343+
else
344+
{
345+
// Add all supported files to in-memory zip from any directories under working directory.
346+
// (recursive)
347+
addSupportedFiles(archiver, configWorkingDirectory, configWorkingDirectory);
348+
}
349+
350+
//////////////////////////////
351+
// Adding config to archive //
352+
//////////////////////////////
306353
// Use a hardcoded name for the config's filename inside the archive.
307354
std::string configFullname = std::string(OCIO_CONFIG_DEFAULT_NAME) +
308355
std::string(OCIO_CONFIG_DEFAULT_FILE_EXT);
309356

357+
std::stringstream ss;
358+
editiableConfig->serialize(ss);
359+
std::string configStr = ss.str();
310360
// Get config string size.
311361
int32_t configSize = (int32_t) configStr.size();
312362

@@ -316,11 +366,13 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
316366
file_info.version_madeby = MZ_VERSION_MADEBY;
317367
file_info.compression_method = MZ_COMPRESS_METHOD_DEFLATE;
318368
file_info.flag = MZ_ZIP_FLAG_UTF8;
319-
file_info.uncompressed_size = configSize;
369+
file_info.uncompressed_size = configSize; // Retrieve and store the config as string.
370+
371+
constexpr uint32_t posix_attrib = 0100644; // File with -rw-r--r-- permissions
372+
constexpr uint32_t msdos_attrib = 0200; // MSDOS equivalent
373+
file_info.external_fa = msdos_attrib;
374+
file_info.external_fa |= (posix_attrib << 16);
320375

321-
//////////////////////////////
322-
// Adding config to archive //
323-
//////////////////////////////
324376
int32_t written = 0;
325377
// Opens an entry in the in-memory zip file for writing.
326378
if (mz_zip_writer_entry_open(archiver, &file_info) == MZ_OK)
@@ -343,16 +395,6 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
343395
throw Exception(os.str().c_str());
344396
}
345397

346-
///////////////////////
347-
// Adding LUT files //
348-
///////////////////////
349-
// Add all supported files to in-memory zip from any directories under working directory.
350-
// (recursive)
351-
if (HasFlag(flags, ARCHIVE_FLAGS_MINIMAL))
352-
addReferencedFiles(archiver, config);
353-
else
354-
addSupportedFiles(archiver, configWorkingDirectory, configWorkingDirectory);
355-
356398
std::ostringstream comment;
357399
comment << "Configuration written by archiveConfig() OCIO: " << GetVersion();
358400
mz_zip_writer_set_comment(archiver, comment.str().c_str());

‎src/apps/ociocheck/main.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,7 @@ int main(int argc, const char **argv)
504504

505505
std::string cacheID;
506506
bool isArchivable = false;
507+
const bool minimal = false;
507508
try
508509
{
509510
LogGuard logGuard;
@@ -512,7 +513,7 @@ int main(int argc, const char **argv)
512513
std::cout << logGuard.output();
513514

514515
cacheID = config->getCacheID();
515-
isArchivable = config->isArchivable();
516+
isArchivable = config->isArchivable(minimal);
516517

517518
// Passed if there are no Error level logs.
518519
StringUtils::StringVec svec = StringUtils::SplitByLines(logGuard.output());

‎tests/cpu/OCIOZArchive_tests.cpp‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)
7373

7474
std::istringstream iss;
7575
iss.str(CONFIG);
76+
const bool minimal = false;
7677

7778
OCIO::ConfigRcPtr cfg;
7879
OCIO_CHECK_NO_THROW(cfg = OCIO::Config::CreateFromStream(iss)->createEditableCopy());
@@ -95,74 +96,74 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)
9596

9697
// Valid search path.
9798
cfg->setSearchPath("luts");
98-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
99+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
99100

100101
cfg->setSearchPath(R"(luts/myluts1)");
101-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
102+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
102103

103104
cfg->setSearchPath(R"(luts\myluts1)");
104-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
105+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
105106

106107
// Valid Search path starting with "./" or ".\".
107108
cfg->setSearchPath(R"(./myLuts)");
108-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
109+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
109110

110111
cfg->setSearchPath(R"(.\myLuts)");
111-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
112+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
112113

113114
// Valid search path starting with "./" or ".\" and a context variable.
114115
cfg->setSearchPath(R"(./$SHOT/myluts)");
115-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
116+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
116117

117118
cfg->setSearchPath(R"(.\$SHOT\myluts)");
118-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
119+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
119120

120121
cfg->setSearchPath(R"(luts/$SHOT)");
121-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
122+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
122123

123124
cfg->setSearchPath(R"(luts/$SHOT/luts1)");
124-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
125+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
125126

126127
cfg->setSearchPath(R"(luts\$SHOT)");
127-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
128+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
128129

129130
cfg->setSearchPath(R"(luts\$SHOT\luts1)");
130-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
131+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
131132

132133
/*
133134
* Illegal scenarios
134135
*/
135136

136137
// Illegal search path starting with "..".
137138
cfg->setSearchPath(R"(luts:../luts)");
138-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
139+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
139140

140141
cfg->setSearchPath(R"(luts:..\myLuts)");
141-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
142+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
142143

143144
// Illegal search path starting with a context variable.
144145
cfg->setSearchPath(R"(luts:$SHOT)");
145-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
146+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
146147

147148
// Illegal search path with absolute path.
148149
cfg->setSearchPath(R"(luts:/luts)");
149-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
150+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
150151

151152
cfg->setSearchPath(R"(luts:/$SHOT)");
152-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
153+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
153154

154155
#ifdef _WIN32
155156
cfg->clearSearchPaths();
156157
cfg->addSearchPath(R"(C:\luts)");
157-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
158+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
158159

159160
cfg->clearSearchPaths();
160161
cfg->addSearchPath(R"(C:\)");
161-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
162+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
162163

163164
cfg->clearSearchPaths();
164165
cfg->addSearchPath(R"(C:\$SHOT)");
165-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
166+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
166167
#endif
167168
}
168169

@@ -181,7 +182,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)
181182
cs->setTransform(ft, OCIO::COLORSPACE_DIR_TO_REFERENCE);
182183
cfg->addColorSpace(cs);
183184

184-
OCIO_CHECK_EQUAL(isArchivable, cfg->isArchivable());
185+
OCIO_CHECK_EQUAL(isArchivable, cfg->isArchivable(minimal));
185186

186187
cfg->removeColorSpace("csTest");
187188
};

0 commit comments

Comments
 (0)