# from re import I
import gitlab
import requests
import json
from json2html import *
import argparse
import os
from dotenv import load_dotenv
from pandas.io.json import json_normalize
import pandas as pd
from flatten_json import flatten
import csv
parser = argparse.ArgumentParser(description="Get Project, License and Owners from gitlab")
parser.add_argument("--production", action='store_true', help="Use for Production")
parser.add_argument("--developer", action='store_true', help="Use for Developer")
args = parser.parse_args()
load_dotenv()
try:
if args.production:
token = os.getenv('PRODUCTION_TOKEN')
print('Production Mode, connect to server')
headers = {'PRIVATE-TOKEN': token}
gl = gitlab.Gitlab(url='https://gitlab.geant.org', private_token=token)
gl_user = 'https://gitlab.geant.org/api/v4/users'
gl_proj = 'https://gitlab.geant.org/api/v4/projects'
gl_license = 'repository/files/LICENSE?ref='
gl_branch = 'repository/branches'
projects = gl.projects.list(all=True)
elif args.developer:
token = os.getenv('DEVELOPER_TOKEN')
print('Developer Mode, connect to server')
headers = {'PRIVATE-TOKEN': token}
gl = gitlab.Gitlab(url='https://k8s-gitlab.software.geant.org', private_token=token)
gl_user = 'https://k8s-gitlab.software.geant.org/api/v4/users'
gl_proj = 'https://k8s-gitlab.software.geant.org/api/v4/projects'
gl_license = 'repository/files/LICENSE?ref='
gl_branch = 'repository/branches'
projects = gl.projects.list(all=True)
# projects = gl.projects.list(all=True)
else:
print("--production or --developer, please, now exit")
exit()
j_project = {}
j_project['projects'] = {}
c_project = []
for project in projects:
print('Project ID: ', project.attributes['id'])
print('Project Name: ', project.attributes['name_with_namespace'])
# # print('branch', project.attributes['default_branch'])
print('Status:', project.attributes['visibility'])
print('aktywny', project.attributes['last_activity_at'])
print('grupa', project.attributes['namespace']['name'])
# pages = project.wikis.list()
# print(pages)
x = project.attributes['id']
check_b = gl_proj + '/' + str(x)
check_b_response = requests.get(check_b, headers=headers)
check_b_data = check_b_response.text
check_b_answer = json.loads(check_b_data)
if "default_branch" in check_b_answer:
branch = project.attributes['default_branch']
# x = project.attributes['id']
c_link = gl_proj + '/' + str(x) + '/' + gl_license + branch
c_response = requests.get(c_link, headers=headers)
c_data = c_response.text
c_answer = json.loads(c_data)
if "message" in c_answer:
if c_answer['message'] == '404 Commit Not Found':
lic = 'No created repository'
# print("License file: ", c_answer['message'])
else:
lic = c_answer['message']
if "file_name" in c_answer:
print("License file: ", c_answer['file_name'])
lic = c_answer['file_name']
# print('branch', project.attributes['default_branch'])
else:
lic = 'No created repository'
sep = 'T'
full_act = project.attributes['last_activity_at']
last_act = full_act.split(sep, 1)[0]
tmp = {
'project_id': project.attributes['id'],
'project_name': project.attributes['name_with_namespace'],
'project_group': project.attributes['namespace']['name'],
'project_status': project.attributes['visibility'],
'project_license': lic,
'project_last_activity': last_act
}
j_project['projects'][x] = {}
j_project['projects'][x] = tmp
j_project['projects'][x]['owners'] = {}
link = (project.attributes['_links']['members'] + '/all')
response = requests.get(link, headers=headers)
data = response.text
owners = json.loads(data)
c_user_tmp = []
for owner in owners:
# c_user_tmp = []
if owner['access_level'] == 50:
# print('User ID: ', owner['id'])
# print('User Name: ', owner['name'])
# print('Access level: ', owner['access_level'])
y = owner['id']
j_project['projects'][x]['owners'][y] = {}
# c_user_tmp = []
user = gl.users.get(y)
emails = user.emails.list()
if user.email == user.commit_email:
user_tmp = {
'owner_id': owner['id'],
'owner_name': owner['name'],
'owner_mail': user.email,
}
# c_user_tmp.append(owner['name'] + '<' + user.email + '>' + ';')
else:
user_tmp = {
'owner_id': owner['id'],
'owner_name': owner['name'],
'owner_mail': user.email,
'owner_mail_commit': user.commit_email
}
# c_user_tmp.append(owner['name'] + '<' + user.email + '>' + ';')
# c_user_tmp.append(j_project['projects'][x]['owners'][y]['owner_name'] + '<' +
# j_project['projects'][x]['owners'][y]['owner_mail'] + '>' + ';')
j_project['projects'][x]['owners'][y] = user_tmp
print(j_project['projects'][x]['owners'][y]['owner_name'] + '<' +
j_project['projects'][x]['owners'][y]['owner_mail'] + '>' + ';')
c_user_tmp.append(j_project['projects'][x]['owners'][y]['owner_name'] + '<' +
j_project['projects'][x]['owners'][y]['owner_mail'] + '>' + ' ')
print(c_user_tmp)
# c_owners = j_project['projects'][x]['owners']
# c_user = []
# for c_owner in c_owners:
# c_user.append(c_owner['owner_name'] + ':' + c_owner['owner_mail'])
c_project.append([project.attributes['id'], project.attributes['name_with_namespace'],
project.attributes['visibility'], lic, last_act, c_user_tmp[0]])
print(8*"-")
except:
print('Something wrong or Cant connect to GitLab server, check credential')
exit()
new_jproject = json.dumps(j_project, indent=4, ensure_ascii=False)
# print(new_jproject)
with open("gitlab-projects.json", "w", encoding='utf8') as outfile:
outfile.write(new_jproject)
new_jproject_html = json2html.convert(json=new_jproject)
with open("gitlab-projects.html", "w", encoding='utf8') as outfile:
outfile.write(new_jproject_html)
print(c_project)
with open('gitlab-projects.csv', 'w', encoding='UTF8', newline='') as f:
writer = csv.writer(f)
# writer = csv.DictWriter(j_project)
# write the header
# writer.writerow(header)
# write multiple rows
writer.writerows(c_project)
# for project in projects:
# print(new_csv)
print("This is the end...")