Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fc 1786 2 av #2742

Merged
merged 3 commits into from
Oct 30, 2024
Merged

Fc 1786 2 av #2742

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8d7514f
Update of the validation process step 2
AnneV-Learn Jun 28, 2024
e4b2ed6
Merge branch 'master' into fc_1786_2_av
AnneV-Learn Jul 19, 2024
c1b324f
Once a POB is Propagated for a CSV record the 'Propagate Alternative …
AnneV-Learn Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions app/controllers/freecen_csv_entries_controller.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,10 @@ def accept
@freecen_csv_entry.update_attributes(warning_messages: '', record_valid: 'true')

@freecen_csv_entry.remove_flags if @freecen_csv_entry.flag

@freecen_csv_file = @freecen_csv_entry.freecen_csv_file
@freecen_csv_file.update_total_warning_messages
session[:propagate_alternate] = @freecen_csv_entry.id unless verbatim_place_of_birth_matches_place_of_birth(@freecen_csv_entry)
session[:propagate_note] = @freecen_csv_entry.id if @freecen_csv_entry.notes.present?
flash[:notice] = 'The acceptance was successful'
redirect_to(freecen_csv_entry_path(@freecen_csv_entry)) && return
end
Expand Down Expand Up @@ -211,10 +214,11 @@ def propagate_alternate
success, message = @freecen_csv_entry.propagate_alternate
if success
flash[:notice] = 'The propagation of the alternate fields was was successful, the file is now locked against replacement until it has been downloaded.'
session[:propagated_alternate] = session[:propagate_alternate]
else
flash[:notice] = "The propagation of the alternate fields failed because #{message}."
session.delete(:propagate_alternate)
end
session.delete(:propagate_alternate)
redirect_to freecen_csv_entry_path(@freecen_csv_entry)
end

Expand Down Expand Up @@ -253,6 +257,8 @@ def show
session[:current_list_entry] = @freecen_csv_entry.id if @next_list_entry.present? || @previous_list_entry.present?
session[:next_list_entry] = @next_list_entry.id if @next_list_entry.present?
session[:previous_list_entry] = @previous_list_entry.id if @previous_list_entry.present?
session[:propagate_alternate] = @freecen_csv_entry.id unless verbatim_place_of_birth_matches_place_of_birth(@freecen_csv_entry) || @freecen_csv_entry.record_valid == 'false'
session[:propagate_note] = @freecen_csv_entry.id if @freecen_csv_entry.notes.present? && @freecen_csv_entry.record_valid == 'true'
end

def update
Expand Down Expand Up @@ -302,6 +308,15 @@ def update_file_statistics(place)
@freecen_csv_file.save
end

def verbatim_place_of_birth_matches_place_of_birth(entry)

return true if entry.birth_county.blank? && entry.birth_place.blank?

return true if entry.verbatim_birth_county == entry.birth_county && entry.verbatim_birth_place == entry.birth_place

false
end

private

def freecen_csv_entry_params
Expand Down
14 changes: 13 additions & 1 deletion app/controllers/freecen_csv_files_controller.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,19 @@ def validate
message = 'File has error messages so cannot be validated'
else
file.update(validation: true)
message = 'File is now ready for validation'
if params[:validate][:prevalidate] == 'yes'
get_user_info_from_userid
user = UseridDetail.id(@userid).first
p "Starting rake task for #{user.userid} CSV File #{file.file_name} in #{file.chapman_code}"
logger.warn("FREECEN:CSV_PREVALIDATE: Starting rake task for #{user.userid} CSV File #{file.file_name} in #{file.chapman_code}")
pid1 = spawn("bundle exec rake freecen:csv_prevalidate[#{file.file_name},#{user.userid}]")
logger.warn("FREECEN:CSV_PREVALIDATE: rake task for #{pid1}")

message = 'The background job that pre-validates the file has been started. You will receive an email when it has been completed. File will then be ready for validation. '
message += 'It has been locked so it must be downloaded before it can be replaced. Please download after pre-validation has completed.'
else
message = 'File is now ready for validation.'
end
end
flash[:notice] = message
redirect_to freecen_csv_file_path(file)
Expand Down
2 changes: 2 additions & 0 deletions app/helpers/freecen_csv_entries_helper.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ def accept_entry
end

def propagate_alternate
return if session[:propagate_alternate] == session[:propagated_alternate]

if session[:propagate_alternate].present? && session[:propagate_alternate] == @freecen_csv_entry.id && @year != '1841'
link_to 'Propagate Alternate Fields', propagate_alternate_freecen_csv_entry_path(@freecen_csv_entry), method: :get, class: "btn btn--small",
title: 'Propagates the alternate fields to entries with the same verbatim POB fields as this entry',
Expand Down
8 changes: 3 additions & 5 deletions app/helpers/freecen_csv_files_helper.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ def edit_freecen_file

def download_freecen_file
link_to 'Download file', download_freecen_csv_file_path(@freecen_csv_file),
data: { confirm: 'Are you sure you want to download these entries?' }, method: :get, class: 'btn btn--small', title: 'Download a copy of the file to your computer. This download WILL INCLUDE and on line changes. Downloading will remove any locks. It will be be stored in your downloads folder'
data: { confirm: 'Are you sure you want to download these entries?' }, method: :get, class: 'btn btn--small', title: 'Download a copy of the file to your computer. This download WILL INCLUDE any online changes. Downloading will remove any locks. It will be be stored in your downloads folder'
end

def convert_header
Expand Down Expand Up @@ -106,11 +106,9 @@ def accept_warnings
title: 'Accepts all warning messages.'
end


def validate_freecen_file
validation_heading = @freecen_csv_file.validation ? 'Validation under way' : 'Commence validation'
link_to "#{validation_heading}", set_validation_freecen_csv_file_path(@freecen_csv_file), class: 'btn btn--small', method: :get,
title: 'Validation of file', data: { confirm: 'Are you sure you want to commence validation of the file?' }
link_to "Validation under way", set_validation_freecen_csv_file_path(@freecen_csv_file), class: 'btn btn--small', method: :get,
title: 'Validation of file', data: { confirm: 'Are you sure - validation is already under way?' }
end

def incorporate_freecen_file
Expand Down
4 changes: 4 additions & 0 deletions app/models/freecen_csv_entry.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1815,6 +1815,8 @@ def propagate?(parameters)

return true if parameters[:birth_place].present? && parameters[:birth_place] != birth_place

return true if birth_county != verbatim_birth_county || birth_place != verbatim_birth_place

false
end

Expand All @@ -1825,6 +1827,8 @@ def propagate_note?(parameters)

return true if parameters[:notes].present? && parameters[:notes] != notes

return true if notes.present?

false
end

Expand Down
47 changes: 47 additions & 0 deletions app/models/freecen_csv_file.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1273,6 +1273,12 @@ def update_messages_and_lock(original_warnings, original_errors, new_warnings, n
update_attributes(locked_by_transcriber: true, total_errors: new_number_errors, total_warnings: new_number_warnings)
end

def update_total_warning_messages
original_number_warnings = total_warnings
new_number_warnings = original_number_warnings - 1
update_attributes(total_warnings: new_number_warnings)
end

def update_freecen_piece
Freecen2Piece.update_or_create_piece(self)
end
Expand All @@ -1288,12 +1294,19 @@ def calculate_transcriber_name(key)
def write_csv_file(file_location)
header = header_line
header << 'record_valid' if validation && !header_line.include?('record_valid')
header << 'pob_valid' if validation && !header_line.include?('pob_valid')
header << 'non_pob_valid' if validation && !header_line.include?('non_pob_valid')
CSV.open(file_location, 'wb', { row_sep: "\r\n" }) do |csv|
csv << header
records = freecen_csv_entries.order_by(_id: 1)
records.each do |rec|
line = []
line = add_fields(line, rec)
if validation
pob_ok = pob_valid(rec)
line << pob_ok
line << non_pob_valid(pob_ok, rec)
end
csv << line
end
end
Expand Down Expand Up @@ -1345,4 +1358,38 @@ def set_total_individuals
update_attributes(total_individuals: number_of_individuals)
end
end

def pob_valid(rec)
result = false
if rec.record_valid == 'true'
result = true
else
has_pob_warning = false
warning_message_parts = rec.warning_messages.split('<br>')
warning_message_parts.each do |part|
has_pob_warning = true if part.include?('Warning:') && part.include?('Birth')
break if has_pob_warning
end
result = true unless has_pob_warning
end
result
end

def non_pob_valid(pob_ok, rec)
result = false
if rec.record_valid == 'true'
result = true
elsif pob_ok == 'true'
result = false
else
has_non_pob_warning = false
warning_message_parts = rec.warning_messages.split('<br>')
warning_message_parts.each do |part|
has_non_pob_warning = true if part.include?('Warning:') && !part.include?('Birth') && !part.include?('Alternate')
break if has_non_pob_warning
end
result = true unless has_non_pob_warning
end
result
end
end
6 changes: 3 additions & 3 deletions app/views/freecen1_vld_files/index.html.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@
<td><%= loaded_at(freecen1_vld_file) %></td>
<td><%= freecen1_vld_file.userid %></td>
<td><%= freecen1_vld_file.transcriber_name %></td>
<% if %w[system_administrator data_manager].include?(session[:role])%>
<% if %w[system_administrator data_manager county_coordinator master_county_coordinator country_coordinator].include?(session[:role])%>
<td><%= pob_val_status(freecen1_vld_file) %></td>
<% if pob_val_status(freecen1_vld_file) != 'All POBs are valid'%>
<td><%= link_to 'Auto Val', auto_validate_pobs_freecen1_vld_file_path(id: freecen1_vld_file.id), method: :get, data: { confirm: 'Are you sure you want to Run Auto POB Validation for '+freecen1_vld_file.file_name+'?'} %></td>
<td><%= link_to 'Pre-Validation', auto_validate_pobs_freecen1_vld_file_path(id: freecen1_vld_file.id), method: :get, data: { confirm: 'Are you sure you want to Run Auto POB Validation for '+freecen1_vld_file.file_name+'?'} %></td>
<% else %>
<td><%= 'Auto Val N/A' %>
<td><%= 'Pre-Validation N/A' %>
<% end %>
<td><%= link_to 'List VLD Entries', entry_csv_download_freecen1_vld_file_path(id: freecen1_vld_file.id), title: 'Download VLD Entries',data: { confirm: 'Are you sure you want to download a list of entries for '+freecen1_vld_file.file_name+'?'}, method: :get %></td>
<% if pob_val_status(freecen1_vld_file) != 'All POBs are valid' && pob_val_status(freecen1_vld_file) != 'Not Started'%>
Expand Down
111 changes: 76 additions & 35 deletions app/views/freecen_csv_files/_show_header.html.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,77 @@
<h2 style="text-align: center"><b><%= file_name(@freecen_csv_file) %> (<%= userid(@freecen_csv_file) %>) in <b><%= @piece.district_name %></b> of <b><%= @piece.chapman_code %></b><br>
</h2>
<p class=" text--center">Videos describing <%= link_to 'this page', "https://youtu.be/GnkkKkVGXs8", target: '_blank' %>, <%= link_to 'replace file', "https://youtu.be/XH-QBKRg9gQ" , target: '_blank' %>, <%= link_to 'download a spreadsheet file', "https://youtu.be/2ekZ_VXmTn0" , target: '_blank' %>, <%= link_to 'download message report', "https://youtu.be/f0l1tlWESII", target: '_blank' %> (all open in new tabs)<br>
</p>
<div style="text-align: center">
<%= replace_freecen_file unless @freecen_csv_file.incorporation_lock %>
<%= convert_header %>
<%= download_freecen_file %>
<%= remove_freecen_file unless @freecen_csv_file.incorporation_lock %>
<%= download_messages %><br>
<%= list_freecen_file_flags %>
<%= list_freecen_file_error_entries %>
<%= list_freecen_file_warning_entries %>
<%= list_freecen_file_information_entries %>
<%= browse_freecen_file_civil_parishes%>
<%= browse_freecen_file_pages%>
<%= browse_freecen_file_dwellings%>
<%= browse_freecen_file_individuals%>
<br>
<% if %w[county_coordinator master_county_coordinator country_coordinator system_administrator data_manager validator executive_director project_manager].include?(session[:role]) %>
<%= change_freecen_file_owner unless @freecen_csv_file.incorporation_lock %>
<%= edit_freecen_file %>
<%= reprocess_freecen_file unless @freecen_csv_file.incorporation_lock %>
<%= accept_warnings unless @freecen_csv_file.incorporation_lock || @freecen_csv_file.total_warnings == 0 %>
<% end%>
<% if %w[county_coordinator master_county_coordinator country_coordinator system_administrator data_manager validator executive_director project_manager].include?(session[:role]) %>
<%= validate_freecen_file unless @freecen_csv_file.incorporation_lock %>
<%= incorporate_freecen_file if @freecen_csv_file.validation%>
<% end%>
<% if ['system_administrator','data_manager'].include?(session[:role]) %>
<%= delete_freecen_file unless @freecen_csv_file.incorporation_lock %>
<% end%>
</div>
<div style="clear:both;"> </div>
<h2 style="text-align: center"><b><%= file_name(@freecen_csv_file) %> (<%= userid(@freecen_csv_file) %>)</b> in <b><%= @piece.district_name %></b> of <b><%= @piece.chapman_code %></b><br>
</h2>
<p class=" text--center">Videos describing <%= link_to 'this page', "https://youtu.be/GnkkKkVGXs8", target: '_blank' %>, <%= link_to 'replace file', "https://youtu.be/XH-QBKRg9gQ" , target: '_blank' %>, <%= link_to 'download a spreadsheet file', "https://youtu.be/2ekZ_VXmTn0" , target: '_blank' %>, <%= link_to 'download message report', "https://youtu.be/f0l1tlWESII", target: '_blank' %> (all open in new tabs)<br>
</p>
<div style="text-align: center">
<%= replace_freecen_file unless @freecen_csv_file.incorporation_lock %>
<%= convert_header %>
<%= download_freecen_file %>
<%= remove_freecen_file unless @freecen_csv_file.incorporation_lock %>
<%= download_messages %><br>
<%= list_freecen_file_flags %>
<%= list_freecen_file_error_entries %>
<%= list_freecen_file_warning_entries %>
<%= list_freecen_file_information_entries %>
<%= browse_freecen_file_civil_parishes%>
<%= browse_freecen_file_pages%>
<%= browse_freecen_file_dwellings%>
<%= browse_freecen_file_individuals%>
<br>
<%= render partial: 'flash_notice' %>
<% if %w[county_coordinator master_county_coordinator country_coordinator system_administrator data_manager validator executive_director project_manager].include?(session[:role]) %>
<%= change_freecen_file_owner unless @freecen_csv_file.incorporation_lock %>
<%= edit_freecen_file %>
<%= reprocess_freecen_file unless @freecen_csv_file.incorporation_lock %>
<%= accept_warnings unless @freecen_csv_file.incorporation_lock || @freecen_csv_file.total_warnings == 0 %>
<% end%>
<% if %w[county_coordinator master_county_coordinator country_coordinator system_administrator data_manager validator executive_director project_manager].include?(session[:role]) %>
<% unless @freecen_csv_file.incorporation_lock || @freecen_csv_file.validation%>
<%= button_tag "Commence Validation", type: 'button', onclick: "ShowValidationDiv()", class: "btn btn--small", id: "validatebutton", title: 'Commence Validation'%>
<% end%>
<%= validate_freecen_file if @freecen_csv_file.validation && !@freecen_csv_file.incorporation_lock %>
<%= incorporate_freecen_file if @freecen_csv_file.validation%>
<% end%>
<% if ['system_administrator','data_manager'].include?(session[:role]) %>
<%= delete_freecen_file unless @freecen_csv_file.incorporation_lock %>
<% end%>
</div>
<div style="clear:both;"> </div>
<div id="validate" style="display:none;">
<%= form_for(:validate, url: set_validation_freecen_csv_file_path(@freecen_csv_file), method: :get, data: { confirm: "Are you sure you want to Commence Validation?"}) do |f| %>
<fieldset class="inputs">
<legend>Commence Validation</legend>
<p>Do you wish to run Pre-validation?</p>
<ol class="grid">
<li class="grid__item one-quarter lap-one-quarter palm-one-whole" id="preval_yes">
<%= f.radio_button :prevalidate, 'yes', :checked =>true %> Yes
</li>
<li class="grid__item one-quarter lap-one-quarter palm-one-whole" id="preval_no">
<%= f.radio_button :prevalidate, 'no', :checked =>false %> No
</li>
<li class="grid__item one-whole lap-one-whole palm-one-whole" id="b_submit"><br>
<%= f.submit 'Submit', :class => "btn btn--small", title: 'mark file as being validated',data: { disable_with: false } %>
</li>
</ol>
<p class="text--left">
<small>Select <b>Yes</b> to mark the file as being validated and start a background job that pre-validates the file. You will receive an email when pre-validation has been completed.</small>
<br>
<small>Select <b>No</b> to mark the file as being validated without running pre-validation.</small>
<br>
<small>Click <b>Cancel Validation</b> button above to Cancel.</small></p>
</fieldset>
<% end %>
</div>
<br>
<%= render partial: 'flash_notice' %>
<script>
function ShowValidationDiv() {
var div = document.getElementById("validate");
var btn = document.getElementById("validatebutton");
if (div.style.display === "none") {
div.style.display = "block";
btn.innerHTML = "Cancel Validation"
} else {
div.style.display = "none";
btn.innerHTML = "Commence Validation"
}
}
</script>
7 changes: 6 additions & 1 deletion app/views/manage_resources/_freecen_release_notes.html.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
<h2>Recent Changes</h2>
<p> Release Notes: June 2024</p>
<ul>
<li>FreeCENMigration#1566: New functionality that allows Data Manager to move search links (census place and POBs) for a place in the Gazetteer.</li>
<li>FreeCENMigration#1785: Update of the validation process step 1.</li>
<li>Fix Actions: Display Syndicate and county coordinators.</li>
</ul>
<p>Release Notes: 15 March 2024</p>
<ul>
<li>FreeCENMigration#1695: Add Freecen2_place_id for Birth Place to Search records.</li>
Expand All @@ -7,7 +13,6 @@
<ul>
<li>FreeCENMigration#1587: Improves validation of vld collection.</li>
</ul>

<p>Release Notes: 26 July 2023</p>
<ul>
<li>FreeCENMigration#1547: Fix age errors.</li>
Expand Down
50 changes: 50 additions & 0 deletions lib/freecen_csv_prevalidator.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
module Freecen
class FreecenCsvPrevalidator

def process_csv_file(csv_file, userid)

invalid_entries = FreecenCsvEntry.where(freecen_csv_file_id: csv_file.id, record_valid: 'false', birth_place: {'$eq'=> nil}, verbatim_birth_place: {'$ne' => nil})
updated_recs = []
invalid_entries.each do |csv_entry|
has_pob_warning = false
warning_message_parts = csv_entry.warning_messages.split('<br>')
warning_message_parts.each do |part|
has_pob_warning = true if part.include?('Warning:') && part.include?('Verbatim Place of Birth') && part.include?('was not found so requires validation')
break if has_pob_warning

end
if has_pob_warning
was_updated = update_from_propagations(csv_entry)
if was_updated
updated_recs << csv_entry.record_number
end
end
end
update_csv_file(csv_file) if updated_recs.size > 0
updated_recs
end

def update_from_propagations(csv_entry)
match_found = false
propagation_match = Freecen1VldEntryPropagation.where(match_verbatim_birth_county: csv_entry.verbatim_birth_county, match_verbatim_birth_place: csv_entry.verbatim_birth_place, scope_year: 'ALL', scope_county: 'ALL').first
if propagation_match.present?
if propagation_match.propagate_pob
warning_message = csv_entry.warning_messages + "Warning: Alternate fields have been adjusted and need review."
csv_entry.update_attributes(birth_county: propagation_match.new_birth_county, birth_place: propagation_match.new_birth_place, warning_messages: warning_message)
end
if propagation_match.propagate_notes
the_notes = csv_entry.notes.blank? ? propagation_match.new_notes : "#{csv_entry.notes} #{propagation_match.new_notes}"
warning_message = csv_entry.warning_messages + "Warning: Notes field has been adjusted and needs review."
csv_entry.update_attributes(notes: the_notes, warning_messages: warning_message)
end
match_found = true
end
match_found
end

def update_csv_file(file)
file.update_attribute(:locked_by_transcriber,true) # lock so that it ca'nt be replaced without being downloaded first.
end

end
end
Loading