Skip to content

Commit 7b3fc93

Browse files
lilithlilith
committed
Add visuals/upload.sh and cargo test to populate the to_upload.txt, fail tests if not all uploaded?
1 parent ed9f957 commit 7b3fc93

File tree

5 files changed

+316
-3
lines changed

5 files changed

+316
-3
lines changed

‎imageflow_core/tests/common/mod.rs‎

Lines changed: 161 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ use imageflow_core::graphics::bitmaps::BitmapWindowMut;
1616
use imageflow_core::{Context, FlowError, ErrorKind};
1717

1818
use s::PixelLayout;
19-
use std::collections::BTreeMap;
19+
use std::collections::{BTreeMap};
2020
use std::fs::File;
2121
use std::path::PathBuf;
22-
use std::io::Write;
22+
use std::io::{Write};
2323
use std;
2424
use std::pin::Pin;
2525
use imageflow_core;
@@ -185,6 +185,10 @@ pub fn smoke_test(input: Option<IoTestEnum>, output: Option<IoTestEnum>, securit
185185
pub struct ChecksumCtx{
186186
checksum_file: PathBuf,
187187
url_list_file: PathBuf,
188+
uploaded_index: PathBuf,
189+
missing_index: PathBuf,
190+
missing_everywhere_index: PathBuf,
191+
to_upload_index: PathBuf,
188192
visuals_dir: PathBuf,
189193
#[allow(dead_code)]
190194
cache_dir: PathBuf,
@@ -208,6 +212,10 @@ impl ChecksumCtx{
208212
create_if_missing: true,
209213
checksum_file: visuals.join(Path::new("checksums.json")),
210214
url_list_file: visuals.join(Path::new("images.txt")),
215+
uploaded_index: visuals.join(Path::new("uploaded.txt")),
216+
missing_index: visuals.join(Path::new("missing_on_s3.txt")),
217+
missing_everywhere_index: visuals.join(Path::new("missing_everwhere.txt")),
218+
to_upload_index: visuals.join(Path::new("to_upload.txt")),
211219
url_base: "https://s3-us-west-2.amazonaws.com/imageflow-resources/visual_test_checksums/"
212220
}
213221
}
@@ -227,6 +235,8 @@ impl ChecksumCtx{
227235
let mut f = ::std::fs::File::create(&self.checksum_file).unwrap();
228236
::serde_json::to_writer_pretty(&mut f, map).unwrap();
229237
f.sync_all().unwrap();
238+
239+
230240
// Write the URL list
231241
// We can use this to prefetch required images in the background on CI)
232242
// TODO: add simple script to do this
@@ -235,9 +245,150 @@ impl ChecksumCtx{
235245
let list_contents = map.values().map(|key| self.image_url(key)).join("\n");
236246
f.write_all(list_contents.as_bytes()).unwrap();
237247
f.sync_all().unwrap();
248+
249+
self.track_missing(map.values().map(|v| self.image_name(v)).collect());
250+
238251
Ok(())
239252
}
240253

254+
pub fn verify_all_active_images_uploaded(&self){
255+
let map = self.load_list().unwrap();
256+
if !self.track_missing(map.values().map(|v| self.image_name(v)).collect()){
257+
panic!("Missing images");
258+
}
259+
}
260+
261+
fn track_missing(&self, active_names: Vec<String>) -> bool{
262+
let mut uploaded = self.load_uploaded().unwrap();
263+
let mut missing = active_names.into_iter().filter(|v| !uploaded.contains(v)).collect::<Vec<String>>();
264+
265+
266+
let stored_missing = self.load_missing().unwrap();
267+
268+
// if stored_missing contains all potential missing, and to_upload.txt isn't empty, we can fail fast rather than checking each image again
269+
if missing.iter().all(|v| stored_missing.contains(v)){
270+
let to_upload = self.load_to_upload().unwrap();
271+
if !to_upload.is_empty(){
272+
eprintln!("{} images need to be uploaded to s3 ({} missing), run ./imageflow_core/tests/visuals/upload.sh and delete to_upload.txt", to_upload.len(), missing.len());
273+
return false; // We're in a "missing" state, we can just stop now.
274+
}
275+
}
276+
277+
self.record_missing(&missing);
278+
279+
let probably_missing = missing.iter().map(|v| v.clone()).collect::<Vec<String>>();
280+
281+
let mut nowhere = Vec::new();
282+
let mut to_upload = Vec::new();
283+
284+
for name in probably_missing {
285+
let remote_url = self.image_url(&name);
286+
print!("Checking {} ...", remote_url);
287+
if let Err(e) = get_url_bytes_with_retry(&remote_url){
288+
// red text
289+
println!("\x1b[31mFAILED\x1b[0m");
290+
eprintln!("\x1b[31m{:?}\x1b[0m", e);
291+
292+
let local_path = self.image_path(&name);
293+
if local_path.exists(){
294+
eprintln!("Found {} locally, adding to the to_upload list", name);
295+
to_upload.push(name.clone());
296+
}else{
297+
nowhere.push(name.clone());
298+
eprintln!("===== {} not found locally or on s3! =====", name);
299+
}
300+
}else{
301+
// green text
302+
println!("\x1b[32mFound {}, removing from missing list. Url: {} \x1b[0m", name, remote_url);
303+
let index = missing.iter().position(|v| v == &name).unwrap();
304+
uploaded.push(name);
305+
missing.remove(index);
306+
self.save_uploaded(&uploaded).unwrap();
307+
self.record_missing(&missing);
308+
}
309+
}
310+
if !missing.is_empty(){
311+
eprintln!("See {} for list of images missing from s3", self.missing_index.display());
312+
}
313+
314+
if !nowhere.is_empty(){
315+
eprintln!("\x1b[31m!!!! {} images are missing both locally and on s3!\x1b[0m", nowhere.len());
316+
eprintln!("\x1b[31mSee {} for list of actively needed images that are missing both locally and on s3!\x1b[0m", self.missing_everywhere_index.display());
317+
for name in &nowhere {
318+
println!("Missing from S3 and locally: {} from s3 {}", name, self.image_url(&name));
319+
}
320+
}
321+
if !to_upload.is_empty(){
322+
323+
// yellow text
324+
eprintln!("\x1b[33mSee {} for list of images that are present locally but missing from s3\x1b[0m", self.missing_everywhere_index.display());
325+
for name in &to_upload {
326+
println!("Missing from S3 but present locally: {} from s3 {}", name, self.image_url(&name));
327+
}
328+
self.record_to_upload(&to_upload);
329+
330+
}
331+
if missing.is_empty() && nowhere.is_empty() && to_upload.is_empty(){
332+
println!("All actively used images are uploaded!");
333+
return true;
334+
}else{
335+
// red text
336+
eprintln!("\x1b[31mUploads not complete. Run ./imageflow_core/tests/visuals/upload.sh to upload missing images\x1b[0m");
337+
return false;
338+
}
339+
340+
}
341+
342+
fn load_missing(&self) -> Result<Vec<String>,()>{
343+
self.load_lines(&self.missing_index)
344+
}
345+
346+
fn record_missing(&self, missing: &[String]){
347+
self.save_lines(&self.missing_index, missing).unwrap();
348+
}
349+
350+
fn load_to_upload(&self) -> Result<Vec<String>,()>{
351+
self.load_lines(&self.to_upload_index)
352+
}
353+
fn record_to_upload(&self, names: &[String]){
354+
self.save_lines(&self.to_upload_index, names).unwrap();
355+
}
356+
357+
fn load_uploaded(&self) -> Result<Vec<String>,()>{
358+
self.load_lines(&self.uploaded_index)
359+
}
360+
361+
fn load_lines(&self, path: &Path) -> Result<Vec<String>,()>{
362+
if path.exists() {
363+
let contents = std::fs::read_to_string(path).unwrap();
364+
let mut lines = contents.lines().collect::<Vec<&str>>();
365+
// remove final empty line
366+
if lines.last() == Some(&""){
367+
lines.pop();
368+
}
369+
let set: Vec<String> = lines.into_iter().map(|v| v.to_owned()).collect();
370+
371+
Ok(set)
372+
}else{
373+
Ok(Vec::new())
374+
}
375+
}
376+
377+
fn save_uploaded(&self, set: &Vec<String>) -> Result<(),()>{
378+
self.save_lines(&self.uploaded_index, set)
379+
}
380+
381+
382+
fn save_lines(&self, path: &Path, lines: &[String]) -> Result<(),()>{
383+
let mut f = ::std::fs::File::create(path).unwrap();
384+
use self::itertools::Itertools;
385+
let list_contents = lines.iter().join("\n");
386+
f.write_all(list_contents.as_bytes()).unwrap();
387+
// write final newline
388+
f.write_all("\n".as_bytes()).unwrap();
389+
f.sync_all().unwrap();
390+
Ok(())
391+
}
241392

242393
/// Get the stored result checksum for a named test
243394
#[allow(unused_variables)]
@@ -277,6 +428,14 @@ impl ChecksumCtx{
277428

278429
self.visuals_dir.as_path().join(Path::new(&name))
279430
}
431+
pub fn image_name(&self, checksum: &str) -> String{
432+
if !checksum.contains("."){
433+
format!("{}.png", checksum)
434+
}else{
435+
format!("{}", checksum)
436+
}
437+
}
438+
280439

281440
pub fn image_path_string(&self, checksum: &str) -> String{
282441
self.image_path(checksum).into_os_string().into_string().unwrap()

‎imageflow_core/tests/visuals.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1972,3 +1972,10 @@ fn test_idct_spatial_no_gamma(){
19721972
//}
19731973

19741974

1975+
1976+
1977+
#[test]
1978+
fn zz_verify_all_checksum_files_uploaded(){
1979+
let ctx = ChecksumCtx::visuals();
1980+
ctx.verify_all_active_images_uploaded();
1981+
}

‎imageflow_core/tests/visuals/upload.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
# Uploads files from to_upload.txt to s3
4+
# https://s3-us-west-2.amazonaws.com/imageflow-resources/visual_test_checksums/
5+
# to_upload.txt contains the filenames
6+
# Loop through to_upload.txt and upload each file to s3
7+
# The files are in the same directory as to_upload.txt (tests/visuals) and upload.sh
8+
# Regardless of working directory, upload.sh will upload the files from the same directory as to_upload.txt
9+
10+
# subshell
11+
12+
# if aws command is present, upload files to s3
13+
14+
echo "You probably want to run aws configure first"
15+
16+
if command -v aws --version &> /dev/null; then
17+
18+
cd $(dirname "$0")
19+
while read -r line; do
20+
echo "Uploading $line"
21+
aws s3 cp ./"$line" s3://imageflow-resources/visual_test_checksums/
22+
done < to_upload.txt
23+
24+
rm missing_on_s3.txt
25+
else
26+
echo "aws not installed, try: sudo snap install aws-cli --classic"
27+
exit 1
28+
fi

‎imageflow_core/tests/visuals/upload_all.sh‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎imageflow_core/tests/visuals/uploaded.txt‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
00196D6A86FF77C80_0BF80F0AE71CD9A63.png
2+
0026D28C52CC20F72.webp
3+
0036B1BD302D16D36_0BF6B81194D5D0D9F.png
4+
0063427E495FBD822_0BF6B81194D5D0D9F.png
5+
0071CBD976FC79E41_0FB5487907981932B.png
6+
00C5CD1E1ADF2628A_0D58EB335590501F3.png
7+
010697250A94CE7C8_0AE4839D1D9B04C57.png
8+
0110973B94D95FAAF_0AB5E1C2525624E58.png
9+
0176FBE641002F3ED_0AE4839D1D9B04C57.png
10+
01864661ED8AB31EF.png
11+
018BC932FB29F2C53_0AF9F040B874D83F7.png
12+
019EAD88244775C0C_05C92341756E39E1D.png
13+
01DD6D368FDF435EF_0E685FA15F0460A97.png
14+
023A1A32B1D12004D_0AB5E1BFABF8ED80B.png
15+
026FF8363C5C47780_0BF6B81194D5D0D9F.png
16+
02791C7249C972865_0B92852F636EA711E.png
17+
0285B89DDE072042D_0AE4839D1D9B04C57.png
18+
02BFB031B1C19953B_0AF9F040B874D83F7.png
19+
02C845E344248A731_0BF6B81194D5D0D9F.png
20+
02CF2323FBBBA9D5D_0AF9F04019B0BAF16.png
21+
02DE05B755CD85DEF_0172196390B512E97.png
22+
02F0D57CD8CBE311E_0AF9F040B874D83F7.png
23+
03801FF0EC8490FE3_0BF6B81194D5D0D9F.png
24+
03AE71E563BAA50E1_0BF80F0AE71CD9A63.png
25+
03B9493B2F3806A45.png
26+
03D1AA84D6ED36458.png
27+
03F352E662F908110_01AA36664748E727D.png
28+
03FB3160ED6A86E15_05C9232CFE2672D1C.png
29+
041A5FD590623EB11_0172196390B512E97.png
30+
04549C40B272C69E8_0EED88CCC2F4CD12F.png
31+
0463725A658D46C9F_0BF80F0AE71CD9A63.png
32+
0486F036E6C1D8825_05C9231C6124D62A8.png
33+
04C0143A72CB4EBAA_073B7BE0C1ABF189F.png
34+
04D1EC9C034ADD12F_0BF80F0AE71CD9A63.png
35+
04E59C99F3F7BCE71_0BF6B81194D5D0D9F.png
36+
04FF87FFA9A645517_09D5B46DC8E13B7B4.png
37+
051EA508DFA6C0FDA_0EED88CCC2F4CD12F.png
38+
05218274046A5989B_0BF80F0AE71CD9A63.png
39+
0539F9C3B850149CB_0BF80F0AE71CD9A63.png
40+
054089B96BC0E7A70_0DE3336C466C67DD9.png
41+
0549704430DA253E1_0BF71EF9142619372.png
42+
0564CEC6B87176F00_0BF6B81194D5D0D9F.png
43+
057FA4CBAC9D0E267_0172196390B512E97.png
44+
062DB2532218625B6_0AE4839D1D9B04C57.png
45+
06C1C57656959DBA7_0AF9F040B874D83F7.png
46+
06F2ECAFA6CA86A45_0AE4839D1D9B04C57.png
47+
070AABF88C5D46610_0BF6B81194D5D0D9F.png
48+
070F860839693FDC7.png
49+
07118D8E2CD72C483_0AF9F04019B0BAF16.png
50+
07BB3F1E3526F79C7_0BF6B81194D5D0D9F.png
51+
07D83583A0B0DE00E_0AF9F040B874D83F7.png
52+
082E998208E7201E4_0AE4839D1D9B04C57.png
53+
08A5881EF99D80FD8_0AF9F040B874D83F7.png
54+
08FCF4B04E5770047_0BF6B81194D5D0D9F.png
55+
0907DA4EA24978ADD_0AE4839D1D9B04C57.png
56+
090A2E5CC38966EE9_05C9232CFE2672D1C.png
57+
094AD8123D38FF811.png
58+
099453586C0B30FFE_0BF6B81194D5D0D9F.png
59+
09BF9877BE2CB36AC_0DE3336C466C67DD9.png
60+
09C384EA1762DABEE_0BF6B81194D5D0D9F.png
61+
09CDB90E528C1B9E5_0BF6B81194D5D0D9F.png
62+
09EAC17A52CD098D9_0AF9F04019B0BAF16.png
63+
09F54D063249F69C8_0172196390B512E97.png
64+
09FA1A44E855DFBB0_01D9D2C49A4E782F9.png
65+
0A839287BD1939BE8.png
66+
0A9BB769D578C3CBD_0BF6B81194D5D0D9F.png
67+
0AFEF1F4539220FD1.webp
68+
0AFF835739E386A12_02E471F17F7F9607D.png
69+
0B16D8BA1DA8542C8_0EED88CCC2F4CD12F.png
70+
0B1B9458B228702CA_0AE4839D1D9B04C57.png
71+
0B3D16EB972FE0DBB.png
72+
0B3DBECF5967D7AF4_02E471F17F7F9607D.png
73+
0B42E995F9EF3B17E_0AB6E6EE83BE6CCAF.png
74+
0B43B2563C986473D_0BF80F0AE71CD9A63.png
75+
0B843EE7B6518278E_0AE4839D1D9B04C57.png
76+
0B8CBDC1706C7C1D1_0BF6B81194D5D0D9F.png
77+
0BA80ECA19E36402B_0DE3336C466C67DD9.png
78+
0BA85F7DC751DD69E_0726847C75419A20C.png
79+
0BEBDEBEADEFEF409.png
80+
0BEE361EED2335E51_05C92341756E39E1D.png
81+
0C303E51B9E1650A4_0BF6B81194D5D0D9F.png
82+
0C51C8FBFD7A960E5_0172196390B512E97.png
83+
0C790C6600AFBBADA_0AE4839D1D9B04C57.png
84+
0C862D7FC5A78146F_0AF9F04019B0BAF16.png
85+
0C98752E6DE58648A.png
86+
0CAA4C5C85C19D4D1_0AF9F04019B0BAF16.png
87+
0CB4F94C3BD8CF521_0BF6B81194D5D0D9F.png
88+
0CE86FEE99A0E0F35.png
89+
0CE9B8941AD0C33EE_0AF9F040B874D83F7.png
90+
0CEA7717E76E828C0_0EED88CCC2F4CD12F.png
91+
0D2D3A96EC962A552_05C92341756E39E1D.png
92+
0D6B7D34193494A6C.jpg
93+
0D6BE331C75748CC1_0BF6B81194D5D0D9F.png
94+
0D6BF8E39199C6CD8_0AE4839D1D9B04C57.png
95+
0D6EA2688E7F01BD5_073B7BE0C1ABF189F.png
96+
0D797226A41557C8C_0AF9F04019B0BAF16.png
97+
0D84206BD2DD308DA_05C9231C6124D62A8.png
98+
0D870139306239192_0AF9F04019B0BAF16.png
99+
0DB362A903C79AC30_0BF80F0AE71CD9A63.png
100+
0DC709F50C5148224.jpg
101+
0E4832C8B06698929_05C92341756E39E1D.png
102+
0E528C65475538702_05C9232CFE2672D1C.png
103+
0E5CFC4A0932ACF87_0BF80F0AE71CD9A63.png
104+
0E650925ECEC2E598_0AE4839D1D9B04C57.png
105+
0E680C080BE9CD3A9_01397E6B9AECAB9F4.png
106+
0E694075B7F251FFD_0BF6B81194D5D0D9F.png
107+
0F3D491AA982C264E_0AF9F040B874D83F7.png
108+
0F4E3A057BF6FACF3_0BF6B81194D5D0D9F.png
109+
0F7C3FC9519DD2F52_0AF9F04019B0BAF16.png
110+
0F8973DC36658EE39_0BF6B81194D5D0D9F.png
111+
0FF6D76B9E4752FBE_0AB5E1C2525624E58.png
112+
002E471E06BC07AEC_05B6C838D00A967A0.png
113+
00A30FB780185F509_05B6C838D00A967A0.png
114+
0F0B95C9157E97C32_05B6C838D00A967A0.png
115+
0DFE368480C3B7FE6_05B6C838D00A967A0.png
116+
0AB703A061274DD1D_05B6C838D00A967A0.png
117+
0ACA8330742B2E0F9_05B6C838D00A967A0.png
118+
008FE0CE5633BFB7E_05B6C838D00A967A0.png
119+
062A1F06054C3DF0A_05B6C838D00A967A0.png
120+
0FBC6A5C3930ADEF7.png

0 commit comments

Comments
 (0)