diff --git a/utils/create-task.py b/utils/create-task-email.txt similarity index 57% copy from utils/create-task.py copy to utils/create-task-email.txt index b770676..edfb107 100644 --- a/utils/create-task.py +++ b/utils/create-task-email.txt @@ -1,82 +1,26 @@ -#!/usr/bin/env python - -from phabricator import Phabricator - -phab = Phabricator() -user = phab.user.whoami() -#print(user) - -group_c4s_admin = 'PHID-PROJ-3bgiie5tgoy7syk7ulhy' -group_c4s = 'PHID-PROJ-itwdg3fgxutsrdnqjklb' - -with open('/tmp/size.csv', 'r') as f: - for l in f.readlines(): - repo_info = l.split(',') - #print repo_info[3] - - repo_user = phab.user.query(usernames=[repo_info[3]]) - #print(repo_user[0]['phid']) - - policy_view = phab.policy.create(objectType='TASK', default='deny',policy=[ - { - 'action': 'allow', - 'rule': 'PhabricatorUsersPolicyRule', - 'value': [repo_user[0]['phid']] - }, - { - 'action': 'allow', - 'rule': 'PhabricatorProjectsPolicyRule', - 'value': [group_c4s_admin] - } - ]) - - policy_edit = phab.policy.create(objectType='TASK', default='deny',policy=[ - { - 'action': 'allow', - 'rule': 'PhabricatorProjectsPolicyRule', - 'value': [group_c4s_admin] - } - ]) - - #print(policy_view) - #print(policy_edit) - - repo_email='''Dear %s, +Dear {name}, You are receiving this email because you have a repository on c4science.ch that is bigger than 2GB. -%s +{repo_id} {repo_name}: {repo_size} These repositories disrupt operations of the service as GIT and SVN are designed to manage source code in text format and not well suited to handle large binary files. Everything that can easily be re-generated should not be in a repository, for example machine code and intermediate files, PDFs, processed data sets, etc. An acceptable utilisation is image files if they are not too big (a few MB at most). If you need to store big files (>1MB) in your repository, please use the Git LFS (Large File Storage) service also provided by c4science: https://c4science.ch/w/c4science/lfs/ Here are the actions you need to do on your repositories: - If your repository contains multiple projects, please create new repositories for each project, split your main repository and we will then delete the former repository: https://c4science.ch/w/c4science/split/ - Add .gitignore rules to filter temporary or generated files that should not be added to the repository: https://c4science.ch/w/c4science/gitignore/ - If your repository contains binary files, things get more complicated. As files stay in the history even when you delete them this could potentially make the issue worse as they could appear twice. - If you are the only contributor to this repository: You can use Git BFG to remove the files from the history and force push to remove all traces of the files: https://c4science.ch/w/c4science/lfs/#convert-already-commited - If there are multiple people collaborating on the same repository BFG will be problematic. It rewrites the Git history and will prevent your coworkers from doing pull of the repository as their history would have diverged. You will need to coordinate with your coworkers so that they do a clean git clone after the use of Git BFG. IMPORTANT: If we have no news from you and the repository size is not reduced before August 1st, 2018, we will have to deactivate it. After a grace period of 3 months the repository will be deleted. Training on the use of c4science.ch and in particular the tools mentioned above is available regularly at EPFL and also on-demand for sufficiently large groups. If you have any questions about how to proceed or any comment about the situation, please answer to this email or on the c4science task and we will help you with any concern. Kind Regards, -c4science team''' % (repo_user[0]['realName'], '- %s %s: %s' % (repo_info[1], repo_info[2], repo_info[0])) - - #print repo_email - - task = phab.maniphest.createtask( - title = 'C4science: One of your repository is too big', - description = repo_email, - viewPolicy = policy_view['phid'], - editPolicy = policy_edit['phid'], - projectPHIDs = [group_c4s], - priority = 100, - ccPHIDs = [repo_user[0]['phid']], - ) - +c4science team diff --git a/utils/create-task.py b/utils/create-task.py old mode 100644 new mode 100755 index b770676..e72abd6 --- a/utils/create-task.py +++ b/utils/create-task.py @@ -1,82 +1,71 @@ #!/usr/bin/env python from phabricator import Phabricator +import logging + +logging.basicConfig(level=logging.DEBUG) phab = Phabricator() user = phab.user.whoami() -#print(user) +logging.debug('user=%s' % user) group_c4s_admin = 'PHID-PROJ-3bgiie5tgoy7syk7ulhy' group_c4s = 'PHID-PROJ-itwdg3fgxutsrdnqjklb' with open('/tmp/size.csv', 'r') as f: for l in f.readlines(): - repo_info = l.split(',') - #print repo_info[3] + repo_info = l.replace('\n', '').split(',') + logging.debug('repo_info=%s' % repo_info) + + if len(repo_info) < 4: + logging.error('CSV error=%s' % l) + break repo_user = phab.user.query(usernames=[repo_info[3]]) - #print(repo_user[0]['phid']) + logging.debug('repo_user=%s' % repo_user) policy_view = phab.policy.create(objectType='TASK', default='deny',policy=[ { 'action': 'allow', 'rule': 'PhabricatorUsersPolicyRule', 'value': [repo_user[0]['phid']] }, { 'action': 'allow', 'rule': 'PhabricatorProjectsPolicyRule', 'value': [group_c4s_admin] } ]) policy_edit = phab.policy.create(objectType='TASK', default='deny',policy=[ { 'action': 'allow', 'rule': 'PhabricatorProjectsPolicyRule', 'value': [group_c4s_admin] } ]) - #print(policy_view) - #print(policy_edit) - - repo_email='''Dear %s, - -You are receiving this email because you have a repository on c4science.ch that is bigger than 2GB. - -%s - -These repositories disrupt operations of the service as GIT and SVN are designed to manage source code in text format and not well suited to handle large binary files. Everything that can easily be re-generated should not be in a repository, for example machine code and intermediate files, PDFs, processed data sets, etc. An acceptable utilisation is image files if they are not too big (a few MB at most). - -If you need to store big files (>1MB) in your repository, please use the Git LFS (Large File Storage) service also provided by c4science: https://c4science.ch/w/c4science/lfs/ + logging.debug('policy_view=%s' % policy_view) + logging.debug('policy_edit=%s' % policy_edit) -Here are the actions you need to do on your repositories: + repo_email_txt = open('./create-task-email.txt', 'r').read() + repo_email = repo_email_txt.format( + name = repo_user[0]['realName'], + repo_id = repo_info[1], + repo_name = repo_info[2], + repo_size = repo_info[0]) -- If your repository contains multiple projects, please create new repositories for each project, split your main repository and we will then delete the former repository: https://c4science.ch/w/c4science/split/ -- Add .gitignore rules to filter temporary or generated files that should not be added to the repository: https://c4science.ch/w/c4science/gitignore/ -- If your repository contains binary files, things get more complicated. As files stay in the history even when you delete them this could potentially make the issue worse as they could appear twice. - - If you are the only contributor to this repository: You can use Git BFG to remove the files from the history and force push to remove all traces of the files: https://c4science.ch/w/c4science/lfs/#convert-already-commited - - If there are multiple people collaborating on the same repository BFG will be problematic. It rewrites the Git history and will prevent your coworkers from doing pull of the repository as their history would have diverged. You will need to coordinate with your coworkers so that they do a clean git clone after the use of Git BFG. - -IMPORTANT: If we have no news from you and the repository size is not reduced before August 1st, 2018, we will have to deactivate it. After a grace period of 3 months the repository will be deleted. - -Training on the use of c4science.ch and in particular the tools mentioned above is available regularly at EPFL and also on-demand for sufficiently large groups. - -If you have any questions about how to proceed or any comment about the situation, please answer to this email or on the c4science task and we will help you with any concern. - -Kind Regards, -c4science team''' % (repo_user[0]['realName'], '- %s %s: %s' % (repo_info[1], repo_info[2], repo_info[0])) - - #print repo_email + logging.debug('repo_email=%s' % repo_email) task = phab.maniphest.createtask( title = 'C4science: One of your repository is too big', description = repo_email, viewPolicy = policy_view['phid'], editPolicy = policy_edit['phid'], projectPHIDs = [group_c4s], priority = 100, ccPHIDs = [repo_user[0]['phid']], ) + logging.debug('task=%s' % task) +