Skip to content

Commit fbdae25

Browse files
OSINT-TECHNOLOGIESOSINT-TECHNOLOGIES
authored
Merge pull request #93 from OSINT-TECHNOLOGIES/rolling
Stabilized v1.1.4
2 parents 73ae125 + 068ff85 commit fbdae25

16 files changed

+283
-1761
lines changed

‎datagather_modules/data_assembler.py

Lines changed: 17 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
8282
print(Fore.GREEN + 'Processing social medias gathering' + Style.RESET_ALL)
8383
social_medias = cp.sm_gather(url)
8484
print(Fore.GREEN + 'Processing subdomain analysis' + Style.RESET_ALL)
85-
if report_file_type == 'pdf':
86-
subdomain_mails, sd_socials, subdomain_ip = cp.domains_reverse_research(subdomains, report_file_type)
87-
elif report_file_type == 'xlsx':
85+
if report_file_type == 'xlsx':
8886
subdomain_urls, subdomain_mails, subdomain_ip, sd_socials = cp.domains_reverse_research(subdomains, report_file_type)
8987
elif report_file_type == 'html':
9088
subdomain_mails, sd_socials, subdomain_ip = cp.domains_reverse_research(subdomains, report_file_type)
@@ -95,7 +93,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
9593
print(Fore.GREEN + 'Extracting robots.txt and sitemap.xml' + Style.RESET_ALL)
9694
robots_txt_result = np.get_robots_txt(short_domain, robots_filepath)
9795
sitemap_xml_result = np.get_sitemap_xml(short_domain, sitemap_filepath)
98-
if report_file_type == 'pdf' or report_file_type == 'html':
96+
if report_file_type == 'html':
9997
sitemap_links_status = np.extract_links_from_sitemap(sitemap_links_filepath, sitemap_filepath)
10098
elif report_file_type == 'xlsx':
10199
try:
@@ -113,66 +111,12 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
113111
common_socials[key] = list(set(common_socials[key]))
114112
total_socials = sum(len(values) for values in common_socials.values())
115113
print(Fore.LIGHTMAGENTA_EX + "\n[BASIC SCAN END]\n" + Style.RESET_ALL)
116-
117-
if report_file_type == 'pdf':
114+
if report_file_type == 'xlsx':
118115
if pagesearch_flag.lower() == 'y':
119116
if subdomains[0] != 'No subdomains were found':
120117
to_search_array = [subdomains, social_medias, sd_socials]
121118
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN START: PAGESEARCH]\n" + Style.RESET_ALL)
122119
ps_emails_return, accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter, website_elements_counter, exposed_passwords_counter, keywords_messages_list = normal_search(to_search_array, report_folder, keywords, keywords_flag)
123-
if len(keywords_messages_list) == 0:
124-
keywords_messages_list = ['No keywords were found']
125-
total_links_counter = accessed_links_counter = 0
126-
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN END: PAGESEARCH]\n" + Style.RESET_ALL)
127-
else:
128-
print(Fore.RED + "Cant start PageSearch because no subdomains were detected")
129-
ps_emails_return = ""
130-
accessible_subdomains = files_counter = cookies_counter = api_keys_counter = website_elements_counter = exposed_passwords_counter = total_links_counter = accessed_links_counter = emails_amount = 'No data was gathered because no subdomains were found'
131-
keywords_messages_list = ['No data was gathered because no subdomains were found']
132-
pass
133-
elif pagesearch_flag.lower() == 'si':
134-
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN START: PAGESEARCH SITEMAP INSPECTION]\n" + Style.RESET_ALL)
135-
ps_emails_return, total_links_counter, accessed_links_counter, emails_amount = sitemap_inspection_search(report_folder)
136-
accessible_subdomains = files_counter = cookies_counter = api_keys_counter = website_elements_counter = exposed_passwords_counter = keywords_messages_list = 0
137-
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN END: PAGESEARCH SITEMAP INSPECTION]\n" + Style.RESET_ALL)
138-
elif pagesearch_flag.lower() == 'n':
139-
accessible_subdomains = files_counter = cookies_counter = api_keys_counter = website_elements_counter = exposed_passwords_counter = total_links_counter = accessed_links_counter = emails_amount = keywords_messages_list = 0
140-
ps_emails_return = ""
141-
pass
142-
143-
if dorking_flag == 'none':
144-
dorking_status = 'Google Dorking mode was not selected for this scan'
145-
dorking_file_path = 'Google Dorking mode was not selected for this scan'
146-
else:
147-
dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
148-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
149-
dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))
150-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
151-
152-
if used_api_flag != ['Empty']:
153-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
154-
if 1 in used_api_flag:
155-
api_virustotal_check(short_domain)
156-
if 2 in used_api_flag:
157-
api_securitytrails_check(short_domain)
158-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
159-
else:
160-
pass
161-
162-
data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials,
163-
subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records,
164-
robots_txt_result, sitemap_xml_result, sitemap_links_status,
165-
web_servers, cms, programming_languages, web_frameworks, analytics, javascript_frameworks, ports,
166-
hostnames, cpes, tags, vulns, common_socials, total_socials, ps_emails_return,
167-
accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter,
168-
website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, keywords_messages_list, dorking_status, dorking_file_path]
169-
170-
elif report_file_type == 'xlsx':
171-
if pagesearch_flag.lower() == 'y':
172-
if subdomains[0] != 'No subdomains were found':
173-
to_search_array = [subdomains, social_medias, sd_socials]
174-
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN START: PAGESEARCH]\n" + Style.RESET_ALL)
175-
ps_emails_return, accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter, website_elements_counter, exposed_passwords_counter = normal_search(to_search_array, report_folder, keywords, keywords_flag)
176120
total_links_counter = accessed_links_counter = 0
177121
print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN END: PAGESEARCH]\n" + Style.RESET_ALL)
178122
else:
@@ -190,23 +134,27 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
190134
accessible_subdomains = files_counter = cookies_counter = api_keys_counter = website_elements_counter = exposed_passwords_counter = total_links_counter = accessed_links_counter = emails_amount = 0
191135
pass
192136

193-
if dorking_flag == 'none':
137+
if dorking_flag == 'n':
194138
dorking_status = 'Google Dorking mode was not selected for this scan'
195139
dorking_results = ['Google Dorking mode was not selected for this scan']
196140
else:
197141
dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
198142
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
199143
dorking_status, dorking_results = dp.transfer_results_to_xlsx(table, dp.get_dorking_query(short_domain, dorking_db_path, table))
200-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
144+
print(Fore.LIGHTMAGENTA_EX + f"[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
201145

146+
api_scan_db = []
202147
if used_api_flag != ['Empty']:
203148
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
204149
if 1 in used_api_flag:
205150
api_virustotal_check(short_domain)
151+
api_scan_db.append('VirusTotal')
206152
if 2 in used_api_flag:
207153
api_securitytrails_check(short_domain)
208-
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
154+
api_scan_db.append('SecurityTrails')
155+
print(Fore.LIGHTMAGENTA_EX + f"[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
209156
else:
157+
api_scan_db.append('No')
210158
pass
211159

212160
data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials,
@@ -243,7 +191,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
243191
ps_emails_return = ""
244192
pass
245193

246-
if dorking_flag == 'none':
194+
if dorking_flag == 'n':
247195
dorking_status = 'Google Dorking mode was not selected for this scan'
248196
dorking_file_path = 'Google Dorking mode was not selected for this scan'
249197
else:
@@ -252,16 +200,21 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
252200
dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))
253201
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
254202

203+
api_scan_db = []
255204
if used_api_flag != ['Empty']:
256205
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
257206
if 1 in used_api_flag:
258207
api_virustotal_check(short_domain)
208+
api_scan_db.append('VirusTotal')
259209
if 2 in used_api_flag:
260210
api_securitytrails_check(short_domain)
211+
api_scan_db.append('SecurityTrails')
261212
print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
262213
else:
214+
api_scan_db.append('No')
263215
pass
264216

217+
265218
data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials,
266219
subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records,
267220
robots_txt_result, sitemap_xml_result, sitemap_links_status,
@@ -270,6 +223,6 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
270223
accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter,
271224
website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, keywords_messages_list, dorking_status, dorking_file_path]
272225

273-
report_info_array = [casename, db_casename, db_creation_date, report_folder, ctime, report_file_type, report_ctime]
226+
report_info_array = [casename, db_casename, db_creation_date, report_folder, ctime, report_file_type, report_ctime, api_scan_db]
274227
logging.info(f'### THIS LOG PART FOR {casename} CASE, TIME: {ctime} ENDS HERE')
275228
return data_array, report_info_array

‎dorking/basic_dorking.db

0 Bytes
Binary file not shown.

‎dorking/dorking_handler.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def save_results_to_txt(folderpath, table, queries, pages=10):
106106
f.write("\n")
107107
dorked_query_counter += 1
108108
print(Fore.GREEN + f" Dorking with " + Style.RESET_ALL + Fore.LIGHTCYAN_EX + Style.BRIGHT + f"{dorked_query_counter}/{total_dorks_amount}" + Style.RESET_ALL + Fore.GREEN + " dork" + Style.RESET_ALL, end="\r")
109-
print(Fore.GREEN + "Google Dorking end. Results successfully saved in TXT file\n" + Style.RESET_ALL)
109+
print(Fore.GREEN + "Google Dorking end. Results successfully saved in HTML report\n" + Style.RESET_ALL)
110110
print(Fore.GREEN + f"During Google Dorking with {table.upper()}:")
111111
for query, count in total_results:
112112
if count == 0:

0 commit comments

Comments
 (0)