|
| 1 | +# ═══════════════════════════════════════════════════════════════════════════════ |
| 2 | +# VIBEE Specification — Scientific Reproducibility System |
| 3 | +# ═══════════════════════════════════════════════════════════════════════════════ |
| 4 | +# Code + Data + Environment = Reproducible Science |
| 5 | +# ═════════════════════════════════════════════════════════════════════════════════ |
| 6 | + |
| 7 | +name: reproducibility |
| 8 | +version: "1.0.0" |
| 9 | +language: zig |
| 10 | +module: reproducibility |
| 11 | + |
| 12 | +description: | |
| 13 | + Scientific Reproducibility System verifies and ensures research can be replicated. |
| 14 | + Checks code availability, environment documentation, data provenance, and metadata. |
| 15 | + Generates reproducibility scores and reports missing elements. |
| 16 | + |
| 17 | +types: |
| 18 | + ReproducibilityItem: |
| 19 | + fields: |
| 20 | + name: String |
| 21 | + description: String |
| 22 | + present: Bool |
| 23 | + required: Bool |
| 24 | + path: String |
| 25 | + |
| 26 | + EnvironmentInfo: |
| 27 | + fields: |
| 28 | + has_dockerfile: Bool |
| 29 | + dockerfile_path: String |
| 30 | + image_tag: String |
| 31 | + base_image: String |
| 32 | + compiler_versions: List<String> |
| 33 | + dependency_versions: List<String> |
| 34 | + |
| 35 | + DataProvenance: |
| 36 | + fields: |
| 37 | + datasets_versioned: Bool |
| 38 | + semantic_versioning: Bool |
| 39 | + repository_urls: List<String> |
| 40 | + doi_available: Bool |
| 41 | + hash_checksums: Bool |
| 42 | + raw_data_preserved: Bool |
| 43 | + |
| 44 | + DocumentationStatus: |
| 45 | + fields: |
| 46 | + has_readme: Bool |
| 47 | + has_api_docs: Bool |
| 48 | + has_setup_guide: Bool |
| 49 | + has_experiment_examples: Bool |
| 50 | + has_metadata_docs: Bool |
| 51 | + |
| 52 | + ReproducibilityReport: |
| 53 | + fields: |
| 54 | + overall_score: Float |
| 55 | + code_available: Bool |
| 56 | + environment_ok: Bool |
| 57 | + data_tracked: Bool |
| 58 | + documentation_complete: Bool |
| 59 | + missing_items: List<String> |
| 60 | + recommendations: List<String> |
| 61 | + |
| 62 | + ReproducibilityConfig: |
| 63 | + fields: |
| 64 | + project_path: String |
| 65 | + strict_mode: Bool |
| 66 | + threshold_score: Float |
| 67 | + check_code: Bool |
| 68 | + check_environment: Bool |
| 69 | + check_data: Bool |
| 70 | + check_docs: Bool |
| 71 | + |
| 72 | +constants: |
| 73 | + DEFAULT_THRESHOLD: Float = 0.8 |
| 74 | + PERFECT_SCORE: Float = 1.0 |
| 75 | + MINIMUM_ACCEPTABLE: Float = 0.6 |
| 76 | + CODE_WEIGHT: Float = 0.3 |
| 77 | + ENVIRONMENT_WEIGHT: Float = 0.25 |
| 78 | + DATA_WEIGHT: Float = 0.25 |
| 79 | + DOCUMENTATION_WEIGHT: Float = 0.2 |
| 80 | + |
| 81 | +behaviors: |
| 82 | + - name: checkReproducibility |
| 83 | + given: config: ReproducibilityConfig |
| 84 | + when: User runs `tri repro_check <project>` |
| 85 | + then: Analyze project and return ReproducibilityReport |
| 86 | + test_cases: |
| 87 | + - name: test_perfect_reproducibility |
| 88 | + input: "{\"mock_complete\": true}" |
| 89 | + expected: "{\"score\": 1.0}" |
| 90 | + - name: test_missing_docker |
| 91 | + input: "{\"has_dockerfile\": false}" |
| 92 | + expected: "{\"environment_ok\": false}" |
| 93 | + |
| 94 | + - name: checkCodeAvailability |
| 95 | + given: project_path: String |
| 96 | + when: Verifying code is in open repository |
| 97 | + then: Return boolean + repository URL |
| 98 | + test_cases: |
| 99 | + - name: test_public_repo |
| 100 | + input: "{\"is_public\": true}" |
| 101 | + expected: "{\"available\": true}" |
| 102 | + |
| 103 | + - name: checkEnvironmentSetup |
| 104 | + given: project_path: String |
| 105 | + when: Checking Docker/container environment |
| 106 | + then: Return EnvironmentInfo with Dockerfile status |
| 107 | + test_cases: |
| 108 | + - name: test_dockerfile_exists |
| 109 | + input: "{\"has_dockerfile\": true}" |
| 110 | + expected: "{\"has_dockerfile\": true}" |
| 111 | + |
| 112 | + - name: checkDataProvenance |
| 113 | + given: project_path: String |
| 114 | + when: Verifying data versioning and tracking |
| 115 | + then: Return DataProvenance status |
| 116 | + test_cases: |
| 117 | + - name: test_versioned_data |
| 118 | + input: "{\"versioned\": true}" |
| 119 | + expected: "{\"datasets_versioned\": true}" |
| 120 | + |
| 121 | + - name: checkDocumentation |
| 122 | + given: project_path: String |
| 123 | + when: Assessing documentation completeness |
| 124 | + then: Return DocumentationStatus |
| 125 | + test_cases: |
| 126 | + - name: test_complete_docs |
| 127 | + input: "{\"has_all\": true}" |
| 128 | + expected: "{\"complete\": true}" |
| 129 | + |
| 130 | + - name: calculateReproducibilityScore |
| 131 | + given: code_ok: Bool, env_ok: Bool, data_ok: Bool, docs_ok: Bool |
| 132 | + when: Computing overall score |
| 133 | + then: Return weighted score 0.0-1.0 |
| 134 | + test_cases: |
| 135 | + - name: test_all_true |
| 136 | + input: "{\"all\": true}" |
| 137 | + expected: "{\"score\": 1.0}" |
| 138 | + - name: test_partial |
| 139 | + input: "{\"code\": true, \"env\": true, \"data\": false, \"docs\": true}" |
| 140 | + expected: "{\"score\": 0.75}" |
| 141 | + |
| 142 | + - name: generateRecommendations |
| 143 | + given: report: ReproducibilityReport |
| 144 | + when: Creating improvement suggestions |
| 145 | + then: Return list of actionable recommendations |
| 146 | + test_cases: |
| 147 | + - name: test_docker_recommendation |
| 148 | + input: "{\"missing\": [\"dockerfile\"]}" |
| 149 | + expected: "{\"recommendations\": [\"Create Dockerfile\"]}" |
| 150 | + |
| 151 | + - name: validateDockerfile |
| 152 | + given: dockerfile_path: String |
| 153 | + when: Checking Dockerfile quality |
| 154 | + then: Return validation with pinned versions check |
| 155 | + test_cases: |
| 156 | + - name: test_pinned_versions |
| 157 | + input: "{\"uses_latest\": false}" |
| 158 | + expected: "{\"valid\": true}" |
| 159 | + |
| 160 | + - name: checkTestVectors |
| 161 | + given: project_path: String |
| 162 | + when: Verifying test vectors exist |
| 163 | + then: Return boolean + count of test vectors |
| 164 | + test_cases: |
| 165 | + - name: test_vectors_present |
| 166 | + input: "{\"count\": 10}" |
| 167 | + expected: "{\"available\": true}" |
| 168 | + |
| 169 | + - name: generateMetadataReport |
| 170 | + given: project_path: String |
| 171 | + when: Creating metadata completeness report |
| 172 | + then: Return report with parameter tracking status |
| 173 | + test_cases: |
| 174 | + - name: test_full_metadata |
| 175 | + input: "{\"has_params\": true}" |
| 176 | + expected: "{\"complete\": true}" |
| 177 | + |
| 178 | + - name: exportReproducibilityBadge |
| 179 | + given: score: Float |
| 180 | + when: Generating badge for README |
| 181 | + then: Return SVG badge with score color |
| 182 | + test_cases: |
| 183 | + - name: test_green_badge |
| 184 | + input: "{\"score\": 0.9}" |
| 185 | + expected: "{\"color\": \"green\"}" |
| 186 | + |
| 187 | +integration: |
| 188 | + std.fs: |
| 189 | + module: std.fs |
| 190 | + functions: |
| 191 | + - cwd |
| 192 | + - openFile |
| 193 | + - access |
| 194 | + |
| 195 | + std.mem: |
| 196 | + module: std.mem |
| 197 | + functions: |
| 198 | + - Allocator |
| 199 | + |
| 200 | + tri_context: |
| 201 | + module: tri_context |
| 202 | + functions: |
| 203 | + - getProjectPath |
| 204 | + |
| 205 | + github_commands: |
| 206 | + module: github_commands |
| 207 | + functions: |
| 208 | + - checkRepoVisibility |
| 209 | + |
| 210 | + tri_commands: |
| 211 | + module: tri_commands |
| 212 | + functions: |
| 213 | + - runChatCommand |
| 214 | + |
| 215 | +# ═══════════════════════════════════════════════════════════════════════════════ |
| 216 | +# IMPLEMENTATION NOTES |
| 217 | +# ═══════════════════════════════════════════════════════════════════════════════ |
| 218 | +# |
| 219 | +# 1. Reproducibility Score Calculation: |
| 220 | +# score = (code_ok * 0.3) + (env_ok * 0.25) + (data_ok * 0.25) + (docs_ok * 0.2) |
| 221 | +# Threshold: 0.8 for "reproducible" status |
| 222 | +# |
| 223 | +# 2. Code Availability Check: |
| 224 | +# - Repository is public (GitHub/GitLab) |
| 225 | +# - Has proper LICENSE file |
| 226 | +# - Version control (git tags/releases) |
| 227 | +# |
| 228 | +# 3. Environment Setup: |
| 229 | +# - Dockerfile exists and uses pinned versions (not :latest) |
| 230 | +# - Base image is stable (Ubuntu LTS, debian:stable) |
| 231 | +# - Compiler versions specified |
| 232 | +# - Dependency versions locked |
| 233 | +# |
| 234 | +# 4. Data Provenance: |
| 235 | +# - Datasets versioned (v1.0.0, v1.1.0, ...) |
| 236 | +# - Raw data preserved |
| 237 | +# - Checksums/hashes available |
| 238 | +# - DOI for datasets (Zenodo, Figshare) |
| 239 | +# - Repository URLs documented |
| 240 | +# |
| 241 | +# 5. Documentation: |
| 242 | +# - README with setup instructions |
| 243 | +# - API documentation |
| 244 | +# - Example experiments |
| 245 | +# - Parameter metadata |
| 246 | +# |
| 247 | +# 6. CLI Command: `tri repro_check <project_path>` |
| 248 | +# - Outputs: Score breakdown, missing items, recommendations |
| 249 | +# - Exit code: 0 if score >= threshold, 1 otherwise |
| 250 | +# - Flag: --strict fails on any missing item |
| 251 | +# |
| 252 | +# 7. Badge Generation: |
| 253 | +# - Score >= 0.9: green (Excellent) |
| 254 | +# - Score >= 0.8: yellow (Good) |
| 255 | +# - Score >= 0.6: orange (Fair) |
| 256 | +# - Score < 0.6: red (Needs improvement) |
| 257 | +# |
| 258 | +# ═════════════════════════════════════════════════════════════════════════════════ |
| 259 | +# END OF SPECIFICATION |
| 260 | +# ═════════════════════════════════════════════════════════════════════════════════════════════ |
0 commit comments