File archival and recovery

Create files to play with.

[1]:

echo "aaaaa" > a.txt
echo "bbbbb" > b.txt
echo "ccccc" > c.txt

File compression and archival

[2]:

ls ?.txt

a.txt  b.txt  c.txt

Combine multiple files into single file

[3]:

tar -cvf abc.tar ?.txt

a.txt
b.txt
c.txt

[4]:

ls

abc.tar
a.txt
b.txt
c.txt
data
figs
lsd1.txt
lsd2.txt
lsd3.txt
R00_Review_Basics.ipynb
R00_Review_Basics_Scratch.ipynb
R01_Data_Manipulation.ipynb
R01_Data_Manipulation_Scratch.ipynb
R01_Data_Manipulation_Solutions.ipynb
R01_Manipulating_Data_In_R.ipynb
R02_Tidying_Data_In_R.ipynb
R02_Tidying_Data.ipynb
R02_Tidying_Data_Solutions.ipynb
R03_FileIO.ipynb
R04_Unsupervised_Learning.ipynb
R04_Unsupervised_Learning_Scratch.ipynb
R05_Unsupervised_Learning_More_Examples.ipynb
R06_Graphics_Overview.ipynb
R07_Graphics_Base.ipynb
R08_Graphics_ggplot2.ipynb
R09_Graphics_Exercise.ipynb
R09_Graphics_Exercise_Solutions.ipynb
seqs
Unix01_File_And_Directory.ipynb
Unix01_File_And_Directory_Solutions.ipynb
Unix02_FileIO.ipynb
Unix02_FileIO_Solutions.ipynb
Unix03_File_Storage.ipynb
Unix03_File_Storage_Solutions.ipynb
Unix04_Text_Manipulation.ipynb
Unix04_Text_Manipulation_Solutinos.ipynb
Unix05_Variables.ipynb
Unix05_Variables_Solutions.ipynb
Unix06_Bash_Bioinformatics.ipynb
Unix07_Capstone_Exercise.ipynb
Unix07_Capstone_Exercise_Solutions.ipynb
Unix_Appendix01_Regular_Expressions.ipynb
Unix_Appendix02_Review.ipynb

Compress concatenated file

[5]:

gzip abc.tar

[6]:

ls

abc.tar.gz
a.txt
b.txt
c.txt
data
figs
lsd1.txt
lsd2.txt
lsd3.txt
R00_Review_Basics.ipynb
R00_Review_Basics_Scratch.ipynb
R01_Data_Manipulation.ipynb
R01_Data_Manipulation_Scratch.ipynb
R01_Data_Manipulation_Solutions.ipynb
R01_Manipulating_Data_In_R.ipynb
R02_Tidying_Data_In_R.ipynb
R02_Tidying_Data.ipynb
R02_Tidying_Data_Solutions.ipynb
R03_FileIO.ipynb
R04_Unsupervised_Learning.ipynb
R04_Unsupervised_Learning_Scratch.ipynb
R05_Unsupervised_Learning_More_Examples.ipynb
R06_Graphics_Overview.ipynb
R07_Graphics_Base.ipynb
R08_Graphics_ggplot2.ipynb
R09_Graphics_Exercise.ipynb
R09_Graphics_Exercise_Solutions.ipynb
seqs
Unix01_File_And_Directory.ipynb
Unix01_File_And_Directory_Solutions.ipynb
Unix02_FileIO.ipynb
Unix02_FileIO_Solutions.ipynb
Unix03_File_Storage.ipynb
Unix03_File_Storage_Solutions.ipynb
Unix04_Text_Manipulation.ipynb
Unix04_Text_Manipulation_Solutinos.ipynb
Unix05_Variables.ipynb
Unix05_Variables_Solutions.ipynb
Unix06_Bash_Bioinformatics.ipynb
Unix07_Capstone_Exercise.ipynb
Unix07_Capstone_Exercise_Solutions.ipynb
Unix_Appendix01_Regular_Expressions.ipynb
Unix_Appendix02_Review.ipynb

[7]:

rm abc.tar.gz

In one step

[8]:

tar -czvf abc.tar.gz ?.txt

a.txt
b.txt
c.txt

[9]:

ls

abc.tar.gz
a.txt
b.txt
c.txt
data
figs
lsd1.txt
lsd2.txt
lsd3.txt
R00_Review_Basics.ipynb
R00_Review_Basics_Scratch.ipynb
R01_Data_Manipulation.ipynb
R01_Data_Manipulation_Scratch.ipynb
R01_Data_Manipulation_Solutions.ipynb
R01_Manipulating_Data_In_R.ipynb
R02_Tidying_Data_In_R.ipynb
R02_Tidying_Data.ipynb
R02_Tidying_Data_Solutions.ipynb
R03_FileIO.ipynb
R04_Unsupervised_Learning.ipynb
R04_Unsupervised_Learning_Scratch.ipynb
R05_Unsupervised_Learning_More_Examples.ipynb
R06_Graphics_Overview.ipynb
R07_Graphics_Base.ipynb
R08_Graphics_ggplot2.ipynb
R09_Graphics_Exercise.ipynb
R09_Graphics_Exercise_Solutions.ipynb
seqs
Unix01_File_And_Directory.ipynb
Unix01_File_And_Directory_Solutions.ipynb
Unix02_FileIO.ipynb
Unix02_FileIO_Solutions.ipynb
Unix03_File_Storage.ipynb
Unix03_File_Storage_Solutions.ipynb
Unix04_Text_Manipulation.ipynb
Unix04_Text_Manipulation_Solutinos.ipynb
Unix05_Variables.ipynb
Unix05_Variables_Solutions.ipynb
Unix06_Bash_Bioinformatics.ipynb
Unix07_Capstone_Exercise.ipynb
Unix07_Capstone_Exercise_Solutions.ipynb
Unix_Appendix01_Regular_Expressions.ipynb
Unix_Appendix02_Review.ipynb

[10]:

rm ?.txt

[11]:

ls

abc.tar.gz
data
figs
lsd1.txt
lsd2.txt
lsd3.txt
R00_Review_Basics.ipynb
R00_Review_Basics_Scratch.ipynb
R01_Data_Manipulation.ipynb
R01_Data_Manipulation_Scratch.ipynb
R01_Data_Manipulation_Solutions.ipynb
R01_Manipulating_Data_In_R.ipynb
R02_Tidying_Data_In_R.ipynb
R02_Tidying_Data.ipynb
R02_Tidying_Data_Solutions.ipynb
R03_FileIO.ipynb
R04_Unsupervised_Learning.ipynb
R04_Unsupervised_Learning_Scratch.ipynb
R05_Unsupervised_Learning_More_Examples.ipynb
R06_Graphics_Overview.ipynb
R07_Graphics_Base.ipynb
R08_Graphics_ggplot2.ipynb
R09_Graphics_Exercise.ipynb
R09_Graphics_Exercise_Solutions.ipynb
seqs
Unix01_File_And_Directory.ipynb
Unix01_File_And_Directory_Solutions.ipynb
Unix02_FileIO.ipynb
Unix02_FileIO_Solutions.ipynb
Unix03_File_Storage.ipynb
Unix03_File_Storage_Solutions.ipynb
Unix04_Text_Manipulation.ipynb
Unix04_Text_Manipulation_Solutinos.ipynb
Unix05_Variables.ipynb
Unix05_Variables_Solutions.ipynb
Unix06_Bash_Bioinformatics.ipynb
Unix07_Capstone_Exercise.ipynb
Unix07_Capstone_Exercise_Solutions.ipynb
Unix_Appendix01_Regular_Expressions.ipynb
Unix_Appendix02_Review.ipynb

Recover files

[12]:

tar -xzvf abc.tar.gz

a.txt
b.txt
c.txt

[13]:

ls

abc.tar.gz
a.txt
b.txt
c.txt
data
figs
lsd1.txt
lsd2.txt
lsd3.txt
R00_Review_Basics.ipynb
R00_Review_Basics_Scratch.ipynb
R01_Data_Manipulation.ipynb
R01_Data_Manipulation_Scratch.ipynb
R01_Data_Manipulation_Solutions.ipynb
R01_Manipulating_Data_In_R.ipynb
R02_Tidying_Data_In_R.ipynb
R02_Tidying_Data.ipynb
R02_Tidying_Data_Solutions.ipynb
R03_FileIO.ipynb
R04_Unsupervised_Learning.ipynb
R04_Unsupervised_Learning_Scratch.ipynb
R05_Unsupervised_Learning_More_Examples.ipynb
R06_Graphics_Overview.ipynb
R07_Graphics_Base.ipynb
R08_Graphics_ggplot2.ipynb
R09_Graphics_Exercise.ipynb
R09_Graphics_Exercise_Solutions.ipynb
seqs
Unix01_File_And_Directory.ipynb
Unix01_File_And_Directory_Solutions.ipynb
Unix02_FileIO.ipynb
Unix02_FileIO_Solutions.ipynb
Unix03_File_Storage.ipynb
Unix03_File_Storage_Solutions.ipynb
Unix04_Text_Manipulation.ipynb
Unix04_Text_Manipulation_Solutinos.ipynb
Unix05_Variables.ipynb
Unix05_Variables_Solutions.ipynb
Unix06_Bash_Bioinformatics.ipynb
Unix07_Capstone_Exercise.ipynb
Unix07_Capstone_Exercise_Solutions.ipynb
Unix_Appendix01_Regular_Expressions.ipynb
Unix_Appendix02_Review.ipynb

[14]:

rm abc.tar.gz

Checksums

When working with genomic data, we deal with very large files. There is a small risk that these files will be corrupted over time or during data transfer. To ensure that files are not changed, we use a “checksum” function. This is a function that generates an long, essentially random number called a checksum that represents the contents of the file. When the file contents change, so will the checksum. In theory, there is a very small probability that two different files generate the same checksum, but in practice the probability is too small to worry about.

There are several different algorithms for generating the checksums, and at least 3 Unix commands to do so, but they all work very similarly for our purposes.

The strategy is:

  • Generate and store a checksum together with a data file whose integrity you care about

  • When you use or download the data, re-generate the checksum (using the same algorithm e.g. MD5) and compare with the checksum

[15]:

cat > hello.txt << EOF
1 Hello, bash
2 Hello, again
3 Hello
4 again
EOF

[16]:

cat hello.txt

1 Hello, bash
2 Hello, again
3 Hello
4 again

Different algorithms to generate checkusms

[17]:

cksum hello.txt

1567754519 45 hello.txt

[18]:

md5sum hello.txt

a68554400613f5445c13c57907e976ed  hello.txt

[19]:

sha1sum hello.txt

57eae725420bf0075d17f849cc8e75379bea6eb6  hello.txt

If we alter hello.txt in any way the checksum will be different

[20]:

cat hello.txt

1 Hello, bash
2 Hello, again
3 Hello
4 again

[21]:

md5sum hello.txt > hello.md5

[22]:

cat hello.md5

a68554400613f5445c13c57907e976ed  hello.txt

[23]:

cat > hello.txt << EOF
1 Hello, bash
2 Hello, again
2 Hello
4 again
EOF

[24]:

cat hello.txt

1 Hello, bash
2 Hello, again
2 Hello
4 again

[25]:

md5sum -c hello.md5

hello.txt: FAILED
md5sum: WARNING: 1 computed checksum did NOT match

Working with multiple files

[26]:

md5sum ?.txt > MD5SUM

[27]:

echo "ccacc" > c.txt

[28]:

md5sum -c MD5SUM

a.txt: OK
b.txt: OK
c.txt: FAILED
md5sum: WARNING: 1 computed checksum did NOT match

Exercises

  1. Create an MD5 checksum for all notebooks (files with extension .ipynb) and store as MD5_CHECKSUM

[29]:

md5sum *ipynb > MD5_CHECKSUM
cat MD5_CHECKSUM

62244c839bf75946704624defd31dc2f  R00_Review_Basics.ipynb
d49c2ea6c69d58abefca0bdc74b4945e  R00_Review_Basics_Scratch.ipynb
8dced15d1a9b95debd69bd2e26fced51  R01_Data_Manipulation.ipynb
b5c9f4b88d09728c3d4f0422bd9ffd18  R01_Data_Manipulation_Scratch.ipynb
9e09863c38312eec48fe00b82770215b  R01_Data_Manipulation_Solutions.ipynb
16fd0a8d9d681974995d8140172c8ef0  R01_Manipulating_Data_In_R.ipynb
87f89f814b13174747026ca668f146d6  R02_Tidying_Data_In_R.ipynb
592d775271c019ac59e82ff7f86f510a  R02_Tidying_Data.ipynb
efd0a6fe5a6155b91d1856dd115aa153  R02_Tidying_Data_Solutions.ipynb
54dc64cfcd4d78d970780f858dc7b40f  R03_FileIO.ipynb
39531aae4ada2bbd009b54b46119bf89  R04_Unsupervised_Learning.ipynb
0831bd591b1c1cc2df2367c260a39a7c  R04_Unsupervised_Learning_Scratch.ipynb
8d9eaf96f9d4535dd322d844f460ebda  R05_Unsupervised_Learning_More_Examples.ipynb
a2fe0422524763aa7c84bc0d93d6bb31  R06_Graphics_Overview.ipynb
c01a3e52a658b7ea8f1c09e5ab0a7468  R07_Graphics_Base.ipynb
968d575cbbcdaf939dd690e6e3d13ef7  R08_Graphics_ggplot2.ipynb
c28451b83183c842f1de53511d30180e  R09_Graphics_Exercise.ipynb
701e20acca68964c4321c263316f2068  R09_Graphics_Exercise_Solutions.ipynb
5fce58fcf8e1fb4b07b68240d06679fb  Unix01_File_And_Directory.ipynb
eb62c94ab276c8d3414e436e3ee0505f  Unix01_File_And_Directory_Solutions.ipynb
769ad300f06de14308852deae7b44e97  Unix02_FileIO.ipynb
93110755e331d4ccc384550f8d9ff631  Unix02_FileIO_Solutions.ipynb
66b6a2f6a69c73ee8662a2ac51c20a17  Unix03_File_Storage.ipynb
fd1e66e88ba42d0812dbf76a5c65c972  Unix03_File_Storage_Solutions.ipynb
79d21d0852b2e936095fddb49aa72ee1  Unix04_Text_Manipulation.ipynb
4a5c5af2eea7002938e500eec054f860  Unix04_Text_Manipulation_Solutinos.ipynb
6ccf56814666d43b580cc3abfed152ab  Unix05_Variables.ipynb
6ccf56814666d43b580cc3abfed152ab  Unix05_Variables_Solutions.ipynb
af15edcf6db6201b09b5ea81046a9cbd  Unix06_Bash_Bioinformatics.ipynb
d73b692af61c7614e01e329f0a5eb89a  Unix07_Capstone_Exercise.ipynb
d2c1ad113a4be50fc17c15bb17159975  Unix07_Capstone_Exercise_Solutions.ipynb
cbe7eb9998cb549acd3ac3b0824407ce  Unix_Appendix01_Regular_Expressions.ipynb
d0798a03d71e02b5aa8093ca883a5239  Unix_Appendix02_Review.ipynb

  1. Archive all notebooks (files with extension .ipynb) in this directory ANd the checksum file as notebooks.tar.gz.

[30]:

tar -czvf notebooks.tar.gz *ipynb MD5_CHECKSUM

R00_Review_Basics.ipynb
R00_Review_Basics_Scratch.ipynb
R01_Data_Manipulation.ipynb
R01_Data_Manipulation_Scratch.ipynb
R01_Data_Manipulation_Solutions.ipynb
R01_Manipulating_Data_In_R.ipynb
R02_Tidying_Data_In_R.ipynb
R02_Tidying_Data.ipynb
R02_Tidying_Data_Solutions.ipynb
R03_FileIO.ipynb
R04_Unsupervised_Learning.ipynb
R04_Unsupervised_Learning_Scratch.ipynb
R05_Unsupervised_Learning_More_Examples.ipynb
R06_Graphics_Overview.ipynb
R07_Graphics_Base.ipynb
R08_Graphics_ggplot2.ipynb
R09_Graphics_Exercise.ipynb
R09_Graphics_Exercise_Solutions.ipynb
Unix01_File_And_Directory.ipynb
Unix01_File_And_Directory_Solutions.ipynb
Unix02_FileIO.ipynb
Unix02_FileIO_Solutions.ipynb
Unix03_File_Storage.ipynb
Unix03_File_Storage_Solutions.ipynb
Unix04_Text_Manipulation.ipynb
Unix04_Text_Manipulation_Solutinos.ipynb
Unix05_Variables.ipynb
Unix05_Variables_Solutions.ipynb
Unix06_Bash_Bioinformatics.ipynb
Unix07_Capstone_Exercise.ipynb
Unix07_Capstone_Exercise_Solutions.ipynb
Unix_Appendix01_Regular_Expressions.ipynb
Unix_Appendix02_Review.ipynb
MD5_CHECKSUM

  1. Delete all (files with extension .ipynb).

[31]:

rm *ipynb

  1. Recover the notebooks from the archived file, and check that the files have not changed during the process of tarring and untarring.

[32]:

tar -xzvf notebooks.tar.gz

R00_Review_Basics.ipynb
R00_Review_Basics_Scratch.ipynb
R01_Data_Manipulation.ipynb
R01_Data_Manipulation_Scratch.ipynb
R01_Data_Manipulation_Solutions.ipynb
R01_Manipulating_Data_In_R.ipynb
R02_Tidying_Data_In_R.ipynb
R02_Tidying_Data.ipynb
R02_Tidying_Data_Solutions.ipynb
R03_FileIO.ipynb
R04_Unsupervised_Learning.ipynb
R04_Unsupervised_Learning_Scratch.ipynb
R05_Unsupervised_Learning_More_Examples.ipynb
R06_Graphics_Overview.ipynb
R07_Graphics_Base.ipynb
R08_Graphics_ggplot2.ipynb
R09_Graphics_Exercise.ipynb
R09_Graphics_Exercise_Solutions.ipynb
Unix01_File_And_Directory.ipynb
Unix01_File_And_Directory_Solutions.ipynb
Unix02_FileIO.ipynb
Unix02_FileIO_Solutions.ipynb
Unix03_File_Storage.ipynb
Unix03_File_Storage_Solutions.ipynb
Unix04_Text_Manipulation.ipynb
Unix04_Text_Manipulation_Solutinos.ipynb
Unix05_Variables.ipynb
Unix05_Variables_Solutions.ipynb
Unix06_Bash_Bioinformatics.ipynb
Unix07_Capstone_Exercise.ipynb
Unix07_Capstone_Exercise_Solutions.ipynb
Unix_Appendix01_Regular_Expressions.ipynb
Unix_Appendix02_Review.ipynb
MD5_CHECKSUM

[33]:

md5sum -c MD5_CHECKSUM

R00_Review_Basics.ipynb: OK
R00_Review_Basics_Scratch.ipynb: OK
R01_Data_Manipulation.ipynb: OK
R01_Data_Manipulation_Scratch.ipynb: OK
R01_Data_Manipulation_Solutions.ipynb: OK
R01_Manipulating_Data_In_R.ipynb: OK
R02_Tidying_Data_In_R.ipynb: OK
R02_Tidying_Data.ipynb: OK
R02_Tidying_Data_Solutions.ipynb: OK
R03_FileIO.ipynb: OK
R04_Unsupervised_Learning.ipynb: OK
R04_Unsupervised_Learning_Scratch.ipynb: OK
R05_Unsupervised_Learning_More_Examples.ipynb: OK
R06_Graphics_Overview.ipynb: OK
R07_Graphics_Base.ipynb: OK
R08_Graphics_ggplot2.ipynb: OK
R09_Graphics_Exercise.ipynb: OK
R09_Graphics_Exercise_Solutions.ipynb: OK
Unix01_File_And_Directory.ipynb: OK
Unix01_File_And_Directory_Solutions.ipynb: OK
Unix02_FileIO.ipynb: OK
Unix02_FileIO_Solutions.ipynb: OK
Unix03_File_Storage.ipynb: OK
Unix03_File_Storage_Solutions.ipynb: OK
Unix04_Text_Manipulation.ipynb: OK
Unix04_Text_Manipulation_Solutinos.ipynb: OK
Unix05_Variables.ipynb: OK
Unix05_Variables_Solutions.ipynb: OK
Unix06_Bash_Bioinformatics.ipynb: OK
Unix07_Capstone_Exercise.ipynb: OK
Unix07_Capstone_Exercise_Solutions.ipynb: OK
Unix_Appendix01_Regular_Expressions.ipynb: OK
Unix_Appendix02_Review.ipynb: OK

[ ]: