NixOS · Conni2461 · Nov 30, 2025
diff --git a/src/lib/Hydra/Base/Controller/ListBuilds.pm b/src/lib/Hydra/Base/Controller/ListBuilds.pm
@@ -20,11 +20,11 @@ sub all : Chained('get_builds') PathPart {
 
     $c->stash->{page} = $page;
     $c->stash->{resultsPerPage} = $resultsPerPage;
-    $c->stash->{total} = $c->stash->{allBuilds}->search({finished => 1})->count
+    $c->stash->{total} = $c->stash->{allBuilds}->search({finished => 1, fodcheck => 0})->count
         unless defined $c->stash->{total};
 
     $c->stash->{builds} = [ $c->stash->{allBuilds}->search(
-        { finished => 1 },
+        { finished => 1, fodcheck => 0 },
         { order_by => "stoptime DESC"
         , columns => [@buildListColumns]
         , rows => $resultsPerPage
@@ -50,7 +50,7 @@ sub latest : Chained('get_builds') PathPart('latest') {
     my ($self, $c, @rest) = @_;
 
     my $latest = $c->stash->{allBuilds}->find(
-        { finished => 1, buildstatus => 0 }, { order_by => ["id DESC"], rows => 1 });
+        { finished => 1, buildstatus => 0, fodcheck => 0 }, { order_by => ["id DESC"], rows => 1 });
 
     notFound($c, "There is no successful build to redirect to.") unless defined $latest;
 
@@ -65,7 +65,7 @@ sub latest_for : Chained('get_builds') PathPart('latest-for') {
     notFound($c, "You need to specify a platform type in the URL.") unless defined $system;
 
     my $latest = $c->stash->{allBuilds}->find(
-        { finished => 1, buildstatus => 0, system => $system }, { order_by => ["id DESC"], rows => 1 });
+        { finished => 1, buildstatus => 0, system => $system, fodcheck => 0 }, { order_by => ["id DESC"], rows => 1 });
 
     notFound($c, "There is no successful build for platform `$system' to redirect to.") unless defined $latest;
 
@@ -79,10 +79,10 @@ sub latest_finished : Chained('get_builds') PathPart('latest-finished') {
     my ($self, $c, @rest) = @_;
 
     my $latest = $c->stash->{allBuilds}->find(
-        { finished => 1, buildstatus => 0 },
+        { finished => 1, buildstatus => 0, fodcheck => 0 },
         { order_by => ["id DESC"], rows => 1, join => ["jobsetevalmembers"]
         , where => \
-            "not exists (select 1 from jobsetevalmembers m2 join builds b2 on jobsetevalmembers.eval = m2.eval and m2.build = b2.id and b2.finished = 0)"
+            "not exists (select 1 from jobsetevalmembers m2 join builds b2 on jobsetevalmembers.eval = m2.eval and m2.build = b2.id and b2.finished = 0 and b2.fodcheck = false)"
         });
 
     notFound($c, "There is no successful build to redirect to.") unless defined $latest;

diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm
@@ -56,7 +56,7 @@ sub latestbuilds : Chained('api') PathPart('latestbuilds') Args(0) {
     my $job = $c->request->params->{job};
     my $system = $c->request->params->{system};
 
-    my $filter = {finished => 1};
+    my $filter = {finished => 1, fodcheck => 0};
     $filter->{"jobset.project"} = $project if ! $project eq "";
     $filter->{"jobset.name"} = $jobset if ! $jobset eq "";
     $filter->{job} = $job if !$job eq "";
@@ -127,7 +127,7 @@ sub queue : Chained('api') PathPart('queue') Args(0) {
     my $nr = $c->request->params->{nr};
     error($c, "Parameter not defined!") if !defined $nr;
 
-    my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["priority DESC", "id"]});
+    my @builds = $c->model('DB::Builds')->search({finished => 0, fodcheck => 0}, {rows => $nr, order_by => ["priority DESC", "id"]});
 
     my @list;
     push @list, buildToHash($_) foreach @builds;
@@ -141,7 +141,7 @@ sub queue : Chained('api') PathPart('queue') Args(0) {
 
 sub nrqueue : Chained('api') PathPart('nrqueue') Args(0) {
     my ($self, $c) = @_;
-    my $nrQueuedBuilds = $c->model('DB::Builds')->search({finished => 0})->count();
+    my $nrQueuedBuilds = $c->model('DB::Builds')->search({finished => 0, fodcheck => 0})->count();
     $c->stash->{'plain'} = {
         data => "$nrQueuedBuilds"
     };
@@ -162,7 +162,7 @@ sub nrbuilds : Chained('api') PathPart('nrbuilds') Args(0) {
     my $job = $c->request->params->{job};
     my $system = $c->request->params->{system};
 
-    my $filter = {finished => 1};
+    my $filter = {finished => 1, fodcheck => 0};
     $filter->{"jobset.project"} = $project if ! $project eq "";
     $filter->{"jobset.name"} = $jobset if ! $jobset eq "";
     $filter->{job} = $job if !$job eq "";

diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm
@@ -517,7 +517,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) {
     }
 
     $c->stash->{channelBuilds} = $c->model('DB::Builds')->search(
-        { id => $build->id },
+        { id => $build->id, fodcheck => 0 },
         { join => ["buildoutputs"]
         , '+select' => ['buildoutputs.path', 'buildoutputs.name'], '+as' => ['outpath', 'outname'] });
 }

diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm
@@ -122,7 +122,7 @@ sub overview : Chained('job') PathPart('') Args(0) {
     $c->stash->{template} = 'job.tt';
 
     $c->stash->{lastBuilds} =
-        [ $c->stash->{jobset}->builds->search({ job => $c->stash->{job}, finished => 1 },
+        [ $c->stash->{jobset}->builds->search({ job => $c->stash->{job}, finished => 1, fodcheck => 0 },
             { order_by => 'id DESC', rows => 10, columns => [@buildListColumns] }) ];
 
     $c->stash->{queuedBuilds} = [
@@ -133,7 +133,7 @@ sub overview : Chained('job') PathPart('') Args(0) {
 
     # If this is an aggregate job, then get its constituents.
     my @constituents = $c->model('DB::Builds')->search(
-        { aggregate => { -in => $c->stash->{jobset}->builds->search({ job => $c->stash->{job} }, { columns => ["id"], order_by => "id desc", rows => 15 })->as_query } },
+        { aggregate => { -in => $c->stash->{jobset}->builds->search({ job => $c->stash->{job}, fodcheck => 0 }, { columns => ["id"], order_by => "id desc", rows => 15 })->as_query }, fodcheck => 0 },
         { join => 'aggregateconstituents_constituents',
           columns => ['id', 'job', 'finished', 'buildstatus'],
           +select => ['aggregateconstituents_constituents.aggregate'],
@@ -220,7 +220,7 @@ sub metric : Chained('job') PathPart('metric') Args(1) {
 # Hydra::Base::Controller::ListBuilds needs this.
 sub get_builds : Chained('job') PathPart('') CaptureArgs(0) {
     my ($self, $c) = @_;
-    $c->stash->{allBuilds} = $c->stash->{jobset}->builds->search({ job => $c->stash->{job} });
+    $c->stash->{allBuilds} = $c->stash->{jobset}->builds->search({ job => $c->stash->{job}, fodcheck => 0 });
     $c->stash->{latestSucceeded} = $c->model('DB')->resultset('LatestSucceededForJob')
         ->search({}, {bind => [$c->stash->{jobset}->id, $c->stash->{job}]});
     $c->stash->{channelBaseName} =

diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm
@@ -114,7 +114,7 @@ sub create_jobset : Chained('evalChain') PathPart('create-jobset') Args(0) {
 sub cancel : Chained('evalChain') PathPart('cancel') Args(0) {
     my ($self, $c) = @_;
     requireCancelBuildPrivileges($c, $c->stash->{project});
-    my $n = cancelBuilds($c->model('DB')->schema, $c->stash->{eval}->builds->search_rs({}));
+    my $n = cancelBuilds($c->model('DB')->schema, $c->stash->{eval}->builds->search_rs({fodcheck => 0}));
     $c->flash->{successMsg} = "$n builds have been cancelled.";
     $c->res->redirect($c->uri_for($c->controller('JobsetEval')->action_for('view'), $c->req->captures));
 }
@@ -123,7 +123,7 @@ sub cancel : Chained('evalChain') PathPart('cancel') Args(0) {
 sub restart {
     my ($self, $c, $condition) = @_;
     requireRestartPrivileges($c, $c->stash->{project});
-    my $builds = $c->stash->{eval}->builds->search_rs({ finished => 1, buildstatus => $condition });
+    my $builds = $c->stash->{eval}->builds->search_rs({ finished => 1, buildstatus => $condition, fodcheck => 0 });
     my $n = restartBuilds($c->model('DB')->schema, $builds);
     $c->flash->{successMsg} = "$n builds have been restarted.";
     $c->res->redirect($c->uri_for($c->controller('JobsetEval')->action_for('view'), $c->req->captures));

diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm
@@ -71,7 +71,7 @@ sub begin :Private {
     if (scalar(@args) == 0 || $args[0] ne "static") {
         $c->stash->{nrRunningBuilds} = dbh($c)->selectrow_array(
             "select count(distinct build) from buildsteps where busy != 0");
-        $c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0 })->count();
+        $c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0, fodcheck => 0 })->count();
     }
 
     # Gather the supported input types.
@@ -132,7 +132,7 @@ sub queue_GET {
     $self->status_ok(
         $c,
         entity => [$c->model('DB::Builds')->search(
-            { finished => 0 },
+            { finished => 0, fodcheck => 0 },
             { order_by => ["globalpriority desc", "id"],
             , columns => [@buildListColumns]
             })]
@@ -147,11 +147,11 @@ sub queue_summary :Local :Path('queue-summary') :Args(0) {
     $c->stash->{queued} = dbh($c)->selectall_arrayref(
         "select jobsets.project as project, jobsets.name as jobset, count(*) as queued, min(timestamp) as oldest, max(timestamp) as newest from Builds " .
         "join Jobsets jobsets on jobsets.id = builds.jobset_id " .
-        "where finished = 0 group by jobsets.project, jobsets.name order by queued desc",
+        "where finished = 0 and fodcheck = false group by jobsets.project, jobsets.name order by queued desc",
         { Slice => {} });
 
     $c->stash->{systems} = dbh($c)->selectall_arrayref(
-        "select system, count(*) as c from Builds where finished = 0 group by system order by c desc",
+        "select system, count(*) as c from Builds where finished = 0 and fodcheck = false group by system order by c desc",
         { Slice => {} });
 }
 
@@ -163,7 +163,7 @@ sub status_GET {
     $self->status_ok(
         $c,
         entity => [$c->model('DB::Builds')->search(
-            { "buildsteps.busy" => { '!=', 0 } },
+            { "buildsteps.busy" => { '!=', 0 }, fodcheck => 0 },
             { order_by => ["globalpriority DESC", "id"],
               join => "buildsteps",
               columns => [@buildListColumns, 'buildsteps.drvpath', 'buildsteps.type']
@@ -286,6 +286,18 @@ sub prometheus :Local Args(0) {
 }
 
 
+sub fod_hash_mismatch :Local :Path('fod-hash-mismatch') :Args(0) {
+    my ($self, $c) = @_;
+    $c->stash->{template} = 'fod-hash-mismatch.tt';
+
+    $c->stash->{fod_mismatches} = dbh($c)->selectall_arrayref(
+        "select builds.id, builds.drvpath, builds.buildstatus, buildoutputs.expectedhash, buildoutputs.actualhash " .
+        "from builds inner join buildoutputs on builds.id = buildoutputs.build " .
+        "where builds.buildstatus = 13 and builds.fodcheck = true",
+        { Slice => {} });
+}
+
+
 # Hydra::Base::Controller::ListBuilds needs this.
 sub get_builds : Chained('/') PathPart('') CaptureArgs(0) {
     my ($self, $c) = @_;
@@ -524,28 +536,28 @@ sub search :Local Args(0) {
             },
             { order_by => ["project", "name"], join => ["project"] } ) ];
 
-        $c->stash->{jobs} = [ $c->model('DB::Builds')->search(
+$c->stash->{jobs} = [ $c->model('DB::Builds')->search(
             { "job" => { ilike => "%$query%" }
             , "project.hidden" => 0
             , "jobset.hidden" => 0
             , iscurrent => 1
+            , fodcheck => 0
             },
             {
                 order_by => ["jobset.project", "jobset.name", "job"],
                 join => { "jobset" => "project" },
                 rows => $c->stash->{limit} + 1
-            } )
-        ];
+            } ) ];
 
         # Perform build search in separate queries to prevent seq scan on buildoutputs table.
         $c->stash->{builds} = [ $c->model('DB::Builds')->search(
-            { "buildoutputs.path" => { ilike => "%$query%" } },
+            { "buildoutputs.path" => { ilike => "%$query%" }, fodcheck => 0 },
             { order_by => ["id desc"], join => ["buildoutputs"]
             , rows => $c->stash->{limit}
             } ) ];
 
         $c->stash->{buildsdrv} = [ $c->model('DB::Builds')->search(
-            { "drvpath" => { ilike => "%$query%" } },
+            { "drvpath" => { ilike => "%$query%" }, fodcheck => 0 },
             { order_by => ["id desc"]
             , rows => $c->stash->{limit}
             } ) ];

diff --git a/src/lib/Hydra/Controller/User.pm b/src/lib/Hydra/Controller/User.pm
@@ -461,6 +461,7 @@ sub my_jobs_tab :Chained('dashboard_base') :PathPart('my-jobs-tab') :Args(0) {
         , maintainers => { ilike => "%" . $c->stash->{user}->emailaddress . "%" }
         , "project.enabled" => 1
         , "jobset.enabled" => 1
+        , fodcheck => 0
         },
         { order_by => ["project", "jobset", "job"]
         , join => {"jobset" => "project"}

diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm
@@ -53,6 +53,7 @@ sub getPreviousBuild {
       , 'me.id' =>  { '<' => $build->id }
       , job => $build->job
       , -not => { buildstatus => { -in => [4, 3]} }
+      , fodcheck => 0
       }, { rows => 1, order_by => "me.id DESC" })->single;
 }
 
@@ -67,6 +68,7 @@ sub getNextBuild {
       , jobset_id => $build->get_column('jobset_id')
       , job => $build->get_column('job')
       , 'me.id' =>  { '>' => $build->id }
+      , fodcheck => 0
       }, {rows => 1, order_by => "me.id ASC"});
 
     return $nextBuild;
@@ -84,6 +86,7 @@ sub getPreviousSuccessfulBuild {
       , job => $build->get_column('job')
       , buildstatus => 0
       , 'me.id' =>  { '<' => $build->id }
+      , fodcheck => 0
       }, {rows => 1, order_by => "me.id DESC"});
 
     return $prevBuild;
@@ -105,7 +108,7 @@ sub searchBuildsAndEvalsForJobset {
 
     foreach my $eval (@evals) {
         my @allBuilds = $eval->builds->search(
-            $condition,
+            { %$condition, fodcheck => 0 },
             { columns => ['id', 'job', 'finished', 'buildstatus'] }
         );
 
@@ -310,7 +313,7 @@ sub getLatestFinishedEval {
     my ($eval) = $jobset->jobsetevals->search(
         { hasnewbuilds => 1 },
         { order_by => "id DESC", rows => 1
-        , where => \ "not exists (select 1 from JobsetEvalMembers m join Builds b on m.build = b.id where m.eval = me.id and b.finished = 0)"
+        , where => \ "not exists (select 1 from JobsetEvalMembers m join Builds b on m.build = b.id where m.eval = me.id and b.finished = 0 and b.fodcheck = false)"
         });
     return $eval;
 }

diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm
@@ -152,10 +152,10 @@ sub jobsetOverview_ {
     return $jobsets->search({},
         { order_by => ["hidden ASC", "enabled DESC", "name"]
         , "+select" =>
-          [ "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 0 and a.isCurrent = 1)"
-          , "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 1 and buildstatus <> 0 and a.isCurrent = 1)"
-          , "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 1 and buildstatus = 0 and a.isCurrent = 1)"
-          , "(select count(*) from Builds as a where me.id = a.jobset_id and a.isCurrent = 1)"
+          [ "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 0 and a.isCurrent = 1 and a.fodcheck = false)"
+          , "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 1 and buildstatus <> 0 and a.isCurrent = 1 and a.fodcheck = false)"
+          , "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 1 and buildstatus = 0 and a.isCurrent = 1 and a.fodcheck = false)"
+          , "(select count(*) from Builds as a where me.id = a.jobset_id and a.isCurrent = 1 and a.fodcheck = false)"
           ]
         , "+as" => ["nrscheduled", "nrfailed", "nrsucceeded", "nrtotal"]
         });
@@ -245,8 +245,8 @@ sub getEvalInfo {
     if (defined $nrSucceeded) {
         $nrScheduled = 0;
     } else {
-        $nrScheduled = $eval->builds->search({finished => 0})->count;
-        $nrSucceeded = $eval->builds->search({finished => 1, buildStatus => 0})->count;
+        $nrScheduled = $eval->builds->search({finished => 0, fodcheck => 0})->count;
+        $nrSucceeded = $eval->builds->search({finished => 1, buildStatus => 0, fodcheck => 0})->count;
         if ($nrScheduled == 0) {
             $eval->update({nrsucceeded => $nrSucceeded});
         }

diff --git a/src/lib/Hydra/Schema/Result/Builds.pm b/src/lib/Hydra/Schema/Result/Builds.pm
@@ -538,7 +538,7 @@ sub makeSource {
 sub makeQueries {
     my ($name, $constraint) = @_;
 
-    my $activeJobs = "(select distinct jobset_id, job, system from Builds where isCurrent = 1 $constraint)";
+    my $activeJobs = "(select distinct jobset_id, job, system from Builds where isCurrent = 1 and fodcheck = false $constraint)";
 
     makeSource(
         "LatestSucceeded$name",

diff --git a/src/lib/Hydra/View/TT.pm b/src/lib/Hydra/View/TT.pm
@@ -82,7 +82,7 @@ sub stripSSHUser {
 # evaluation.
 sub jobExists {
     my ($self, $c, $jobset, $jobName) = @_;
-    return defined $jobset->builds->search({ job => $jobName, iscurrent => 1 })->single;
+    return defined $jobset->builds->search({ job => $jobName, iscurrent => 1, fodcheck => 0 })->single;
 }
 
 =head2 linkToProject

diff --git a/src/root/build.tt b/src/root/build.tt
@@ -87,6 +87,8 @@ END;
                 <span class="error">Output limit exceeded</span>
               [% ELSIF step.status == 12 %]
                 <span class="error">Non-determinism detected</span> [% IF step.timesbuilt %] after [% HTML.escape(step.timesbuilt) %] times[% END %]
+              [% ELSIF step.status == 13 %]
+                <span class="error">Hash mismatch</span>
               [% ELSIF step.errormsg %]
                 <span class="error">Failed</span>: <em>[% HTML.escape(step.errormsg) %]</em>
               [% ELSE %]

diff --git a/src/root/common.tt b/src/root/common.tt
@@ -306,6 +306,8 @@ BLOCK renderStatus;
       <span class="error">Output limit exceeded</span>
     [% ELSIF buildstatus == 12 %]
       <span class="error">Non-deterministic build</span>
+    [% ELSIF buildstatus == 13 %]
+      <span class="error">Hash mismatch</span>
     [% ELSE %]
       <span class="error">Aborted</span>
       (Hydra failure; see <a href="#nix-error">below</a>)

diff --git a/src/root/fod-hash-mismatch.tt b/src/root/fod-hash-mismatch.tt
@@ -0,0 +1,41 @@
+[% WRAPPER layout.tt title="FOD Hash Mismatches" %]
+[% PROCESS common.tt %]
+
+[% IF fod_mismatches.size == 0 %]
+
+  <div class="alert alert-info">No FOD hash mismatches found.</div>
+
+[% ELSE %]
+
+  <div class="alert alert-warning">
+    <strong>[% fod_mismatches.size %]</strong> builds with FOD hash mismatches found.
+  </div>
+
+  <table class="table table-striped table-condensed">
+    <thead>
+      <tr>
+        <th>Build ID</th>
+        <th>Derivation Path</th>
+        <th>Expected Hash</th>
+        <th>Actual Hash</th>
+      </tr>
+    </thead>
+    <tbody>
+      [% FOREACH m IN fod_mismatches %]
+        <tr>
+          <td><a href="[% c.uri_for('/build' m.id) %]">[% HTML.escape(m.id) %]</a></td>
+          <td>
+            <a href="[% c.uri_for('/drvpath-history', m.drvpath) %]">
+              <tt>[% HTML.escape(m.drvpath) %]</tt>
+            </a>
+          </td>
+          <td><tt>[% HTML.escape(m.expectedhash) %]</tt></td>
+          <td><tt>[% HTML.escape(m.actualhash) %]</tt></td>
+        </tr>
+      [% END %]
+    </tbody>
+  </table>
+
+[% END %]
+
+[% END %]
diff --git a/src/root/topbar.tt b/src/root/topbar.tt
@@ -37,6 +37,9 @@
     [% INCLUDE menuItem
        uri = c.uri_for(c.controller('Root').action_for('queue_runner_status'))
        title = "Queue Runner Status" %]
+    [% INCLUDE menuItem
+       uri = c.uri_for(c.controller('Root').action_for('fod_hash_mismatch'))
+       title = "FOD Hash Mismatches" %]
   [% END %]
 
   [% IF project %]