Recursively create a file - digitalquake - Nov-02-2016
Hello all I'm new to the world of python.
(please don't bully the scriptkiddo  )
I've wrote script that create folders in a specific location on a RAID, updated every night to the cloud with a cronjob.
However, since the dosent' allow empty "folders", if one is not used will stay empty and won't be uploaded.
That's why I would like to add a recursive "touch" that create a hidden file, like store.oks or something similar, that allow AWSCLI (command line from amazon cloud) to properly update the whole folder structure.
Can someone help me out or point in the right direction, please?
here's a copy of the script
#!/usr/bin/python
print 'Running script'
while True:
import os
aw_directory = "path/9999_testfolder"
for child in os.listdir(aw_directory):
project_path = os.path.join(aw_directory, child)
if os.path.isdir(project_path):
# DATALAB BRANCH
a = os.path.join(project_path, '001_DATALAB')
open('store.oks', 'r').close()
b = os.path.join(project_path, '001_DATALAB/001_Rushes')
c = os.path.join(project_path, '001_DATALAB/001_Rushes/001_video')
d = os.path.join(project_path, '001_DATALAB/001_Rushes/002_audio')
e = os.path.join(project_path, '001_DATALAB/002_Transcodes')
f = os.path.join(project_path, '001_DATALAB/003_Encodes')
g = os.path.join(project_path, '001_DATALAB/004_Artwork-Titles-Graphics')
h = os.path.join(project_path, '001_DATALAB/005_Music-Sound')
# EDITORIAL BRANCH
i = os.path.join(project_path,'002_EDITORIAL')
j = os.path.join(project_path,'002_EDITORIAL/001_offline/')
k = os.path.join(project_path,'002_EDITORIAL/001_offline/001_exports')
l = os.path.join(project_path,'002_EDITORIAL/001_offline/002_assets')
m = os.path.join(project_path,'002_EDITORIAL/002_online')
n = os.path.join(project_path,'002_EDITORIAL/002_online/001_exports')
o = os.path.join(project_path,'002_EDITORIAL/002_online/002_assets')
# ANIMATION BRANCH
p = os.path.join(project_path,'003_ANIMATION')
q = os.path.join(project_path,'003_ANIMATION/001_exports')
r = os.path.join(project_path,'003_ANIMATION/002_assets')
# GRADE BRANCH
s = os.path.join(project_path,'004_GRADE')
t = os.path.join(project_path,'004_GRADE/001_exports')
u = os.path.join(project_path,'004_GRADE/002_assets')
# VFX BRANCH
v = os.path.join(project_path,'005_VFX')
w = os.path.join(project_path,'005_VFX/001_exports')
x = os.path.join(project_path,'005_VFX/002_assets')
# REF&TREATMENTS BRANCH
y = os.path.join(project_path,'006_REF-TREATMENTS')
# DELIVERABLES BRANCH
z = os.path.join(project_path,'007_DELIVERABLES')
aa = os.path.join(project_path,'007_DELIVERABLES/001_WIP')
ab = os.path.join(project_path,'007_DELIVERABLES/001_WIP/001_offline')
ac = os.path.join(project_path,'007_DELIVERABLES/001_WIP/002_animation')
ad = os.path.join(project_path,'007_DELIVERABLES/001_WIP/003_grade')
ae = os.path.join(project_path,'007_DELIVERABLES/001_WIP/004_VFX')
af = os.path.join(project_path,'007_DELIVERABLES/001_WIP/005_online')
ag = os.path.join(project_path,'007_DELIVERABLES/002_Master')
ah = os.path.join(project_path,'007_DELIVERABLES/002_Master/encodes')
# DATALAB VARIABLES
if not os.path.exists(a):
original_umask_a = os.umask(0)
os.makedirs(a, mode=0777)
os.umask(original_umask_a)
if not os.path.exists(b):
original_umask_b = os.umask(0)
os.makedirs(b, mode=0777)
os.umask(original_umask_b)
if not os.path.exists(c):
original_umask_c = os.umask(0)
os.makedirs(c, mode=0777)
os.umask(original_umask_c)
if not os.path.exists(d):
original_umask_d = os.umask(0)
os.makedirs(d, mode=0777)
os.umask(original_umask_d)
if not os.path.exists(e):
original_umask_e = os.umask(0)
os.makedirs(e, mode=0777)
os.umask(original_umask_e)
if not os.path.exists(f):
original_umask_f = os.umask(0)
os.makedirs(f, mode=0777)
os.umask(original_umask_f)
if not os.path.exists(g):
original_umask_g = os.umask(0)
os.makedirs(g, mode=0777)
os.umask(original_umask_g)
if not os.path.exists(h):
original_umask_h = os.umask(0)
os.makedirs(h, mode=0777)
os.umask(original_umask_h)
# EDITORIAL VARIABLES
if not os.path.exists(i):
original_umask_i = os.umask(0)
os.makedirs(i, mode=0777)
os.umask(original_umask_i)
if not os.path.exists(j):
original_umask_j = os.umask(0)
os.makedirs(j, mode=0777)
os.umask(original_umask_j)
if not os.path.exists(k):
original_umask_k = os.umask(0)
os.makedirs(k, mode=0777)
os.umask(original_umask_k)
if not os.path.exists(l):
original_umask_l = os.umask(0)
os.makedirs(l, mode=0777)
os.umask(original_umask_l)
if not os.path.exists(m):
original_umask_m = os.umask(0)
os.makedirs(m, mode=0777)
os.umask(original_umask_m)
if not os.path.exists(n):
original_umask_n = os.umask(0)
os.makedirs(n, mode=0777)
os.umask(original_umask_n)
if not os.path.exists(o):
original_umask_o = os.umask(0)
os.makedirs(o, mode=0777)
os.umask(original_umask_o)
# ANIMATION VARIABLES
if not os.path.exists(p):
original_umask_p = os.umask(0)
os.makedirs(p, mode=0777)
os.umask(original_umask_p)
if not os.path.exists(q):
original_umask_q = os.umask(0)
os.makedirs(q, mode=0777)
os.umask(original_umask_q)
if not os.path.exists(r):
original_umask_r = os.umask(0)
os.makedirs(r, mode=0777)
os.umask(original_umask_r)
# GRADE VARIABLES
if not os.path.exists(s):
original_umask_s = os.umask(0)
os.makedirs(s, mode=0777)
os.umask(original_umask_s)
if not os.path.exists(t):
original_umask_t = os.umask(0)
os.makedirs(t, mode=0777)
os.umask(original_umask_t)
if not os.path.exists(u):
original_umask_u = os.umask(0)
os.makedirs(u, mode=0777)
os.umask(original_umask_u)
# VFX VARIABLES
if not os.path.exists(v):
original_umask_v = os.umask(0)
os.makedirs(v, mode=0777)
os.umask(original_umask_v)
if not os.path.exists(w):
original_umask_w = os.umask(0)
os.makedirs(w, mode=0777)
os.umask(original_umask_w)
if not os.path.exists(x):
original_umask_x = os.umask(0)
os.makedirs(x, mode=0777)
os.umask(original_umask_x)
# REF&TREATMENTS VARIABLES
if not os.path.exists(y):
original_umask_y = os.umask(0)
os.makedirs(y, mode=0777)
os.umask(original_umask_y)
# DELIVERABLES VARIABLES
if not os.path.exists(z):
original_umask_z = os.umask(0)
os.makedirs(z, mode=0777)
os.umask(original_umask_z)
if not os.path.exists(aa):
original_umask_aa = os.umask(0)
os.makedirs(aa, mode=0777)
os.umask(original_umask_aa)
if not os.path.exists(ab):
original_umask_ab = os.umask(0)
os.makedirs(ab, mode=0777)
os.umask(original_umask_ab)
if not os.path.exists(ac):
original_umask_ac = os.umask(0)
os.makedirs(ac, mode=0777)
os.umask(original_umask_ac)
if not os.path.exists(ad):
original_umask_ad = os.umask(0)
os.makedirs(ad, mode=0777)
os.umask(original_umask_ad)
if not os.path.exists(ae):
original_umask_ae = os.umask(0)
os.makedirs(ae, mode=0777)
os.umask(original_umask_ae)
if not os.path.exists(af):
original_umask_af = os.umask(0)
os.makedirs(af, mode=0777)
os.umask(original_umask_af)
if not os.path.exists(ag):
original_umask_ag = os.umask(0)
os.makedirs(ag, mode=0777)
os.umask(original_umask_ag)
if not os.path.exists(ah):
original_umask_ah = os.umask(0)
os.makedirs(ah, mode=0777)
os.umask(original_umask_ah)
Thank you for your time
RE: Recursively create a file - Ofnuts - Nov-02-2016
Strong recommendation: learn about lists and how to iterate them. You code could be around 30 lines...
To make an empty file: just use
open('filename.dat','w').close()(note that unlike "touch" this will overwrite an existing file...)
RE: Recursively create a file - digitalquake - Nov-02-2016
I'll look into it, thank you.
So I can use list for variables (all the leters) for assiging the correct folder and use a loop to check if the correct folder exist?
Can I do a reiteration for have this line of code happening everytime a folder is created? Or do I need to stick it underneath every folder branch to have the file in place?
open('filename.dat','w').close()
RE: Recursively create a file - Ofnuts - Nov-02-2016
Something like this (syntax not checked):
dirs=[
'001_DATALAB',
'001_DATALAB/001_Rushes',
'001_DATALAB/001_Rushes/001_video',
'001_DATALAB/001_Rushes/002_audio'
# add more directories here
]
# Now loop over the directories above, doing the same thing for each:
for dir in dirs:
fulldir=os.path.join(project_path,dir)
if not os.path.exists(fulldir):
os.makedirs(fulldir, mode=0777)
placeholder=os.path.join(fulldir,dir)
open('store.oks', 'w').close() # has to be opened in write mode...
I don't really understand what you are doing with the umask, this seems completely useless
RE: Recursively create a file - digitalquake - Nov-03-2016
Thanks for the hints :)
I think the umask is there because this script write to an active folder shared over network and the scripts has to be run under root or an admin that can write folders with 777 permission in order to make them accessible to different users of the network.
RE: Recursively create a file - wavic - Nov-03-2016
(Nov-03-2016, 10:02 AM)digitalquake Wrote: Thanks for the hints :)
I think the umask is there because this script write to an active folder shared over network and the scripts has to be run under root or an admin that can write folders with 777 permission in order to make them accessible to different users of the network.
It's not safe to set permissions to 777. Instead, create a group, assign the users to this group and set the owner of the file to be user:group and set the corresponding permissions.
RE: Recursively create a file - Ofnuts - Nov-03-2016
(Nov-03-2016, 10:02 AM)digitalquake Wrote: Thanks for the hints :)
I think the umask is there because this script write to an active folder shared over network and the scripts has to be run under root or an admin that can write folders with 777 permission in order to make them accessible to different users of the network.
If you use the mode explicitly and never change the umask this is pointless. What you can do is change the umask once for all at the beginning to 0000 so that it doesn't restrict the rights on the files and directories you create (0666 for files and 0777 for directories). Which means your code could be a lot simpler.
RE: Recursively create a file - digitalquake - Nov-03-2016
Our setup use netatalk and smb share so people have access only to limited folders, and 0777 is handy for them so they can read and write into the folders but not messup with other projects. Maybe a swtich to 0666 is not a bad idea.
RE: Recursively create a file - Ofnuts - Nov-03-2016
(Nov-03-2016, 04:29 PM)digitalquake Wrote: Our setup use netatalk and smb share so people have access only to limited folders, and 0777 is handy for them so they can read and write into the folders but not messup with other projects. Maybe a swtich to 0666 is not a bad idea.
Keep 777 for folders. The "execute" bit is important. If reset, you cannot access anything under that directory.
RE: Recursively create a file - digitalquake - Nov-09-2016
(Nov-03-2016, 10:26 PM)Ofnuts Wrote: (Nov-03-2016, 04:29 PM)digitalquake Wrote: Our setup use netatalk and smb share so people have access only to limited folders, and 0777 is handy for them so they can read and write into the folders but not messup with other projects. Maybe a swtich to 0666 is not a bad idea.
Keep 777 for folders. The "execute" bit is important. If reset, you cannot access anything under that directory.
I see. thanks for the tip!
|
