Как мы парсили информацию об уязвимостях

# parse bulletine NKCKI------------------------------------------------------------------------------------------------
def get_cve_list(name_list):
    cve_list = []
    cve_list_repeat = []
    for name in name_list:
        pdf_file = open(f'{PATH}/{name}', 'rb')
        read_pdf = PyPDF2.PdfFileReader(pdf_file)
        number_of_pages = read_pdf.getNumPages()
        data = ''
        for i in range(number_of_pages):
            page = read_pdf.getPage(i)
            page_content = page.extractText()
            data1 = json.dumps(page_content)
            data += data1

        rez = ''
        for n, item in enumerate(data, start=0):
            if item != '\\':
                rez += data[n]

        result = ''
        for n, item in enumerate(rez, start=0):
            if item != 'n':
                result += rez[n]
        rez1 = result.replace(" ", "")

        regex = re.findall(r'CVE-\d{4}-\d{4,8}', rez1)
        for cve in regex:
            cve_list_repeat.append(cve)
    for item in cve_list_repeat:
        if item not in cve_list:
            cve_list.append(item)
    return cve_list


def get_links_for_bulletine():
    links = []
    for page in range(3):
        r = requests.get("https://safe-surf.ru/specialists/bulletins-nkcki/?PAGEN_1={}".format(page))
        soup = BeautifulSoup(r.text, "html.parser")
        for vuln in soup.find_all("div", "blockBase blockBulletine"):
            bulletin_pdf_url = "https://safe-surf.ru{}".format(vuln.find('h4').find('a')['href'])
            links.append(bulletin_pdf_url)
    return links


def create_pdf_file(links):
    for str in links:
        get = requests.get(str)
        name = str.replace("https://safe-surf.ru/upload/VULN/", "")
        check_file = os.path.exists(f'{PATH}/{name}')
        if check_file == 0:
            with open(f'{PATH}/{name}', 'wb') as f:
                f.write(get.content)
        else:
            print(f'file {name} already exists')


def get_name_list(path):
    name_list = os.listdir(path)
    return name_list


def remove_pdf_file(path):
    for item in path:
        os.remove(item)

links = get_links_for_bulletine()
create_pdf_file(links)
name_list = get_name_list(PATH)  # Получение имен скаченных файлов для парсинга
cve_line = get_cve_list(name_list)  # Получение списка cve из биллютеня НКЦКИ
print(cve_line)

# remove pdf buffer--------------------------------------------------------------------------------------------------- 
time.sleep(1)
path_to_remove_pdf = []
for item in name_list:
    path_to_remove_pdf.append(f'{PATH}/{item}')
remove_pdf_file(path_to_remove_pdf)

# parse opencve.io---------------------------------------------------------------------------------------------------- 
def parsing_opencve():
    url1 = 'https://www.opencve.io/login/'
    url2 = 'https://www.opencve.io/login'
    csrf_token = ''
    s = requests.Session()
    response = s.get(url1)
    soup = BeautifulSoup(response.text, 'lxml')

    # Get CSRF
    for a in soup.find_all('meta'):
        if 'name' in a.attrs:
            if a.attrs['name'] == 'csrf-token':
                csrf_token = a.attrs['content']

    # Authentication
    s.post(
        url2,
        data={
            'username': USERNAME,
            'password': PASSWORD,
            'csrf_token': csrf_token,
        },
        headers={'referer': 'https://www.opencve.io/login'},
        verify=False
    )
    # Get new CVE
    cve_line = []
    for page_num in range(1, 20):
        pagination = f'https://www.opencve.io/?page={page_num}'
        resp = s.get(pagination)
        parse = BeautifulSoup(resp.text, 'lxml')
        for cve in parse.find_all('h3', class_='timeline-header'):
            index = cve.text.find('has changed')
            if index == -1:
                cve_line.append(cve.text.replace(' is a new CVE', ''))

    cve_line_no_replic = []
    for item in cve_line:
        if item not in cve_line_no_replic:
            cve_line_no_replic.append(item[:-1])
    return cve_line_no_replic

cve_line = parsing_opencve()  # Получение списка новых cve с сайта opencve.io
print(cve_line)

# parse cvetrends.com--------------------------------------------------------------------------------------------------
def get_top_cve_list():
    link = 'https://cvetrends.com/api/cves/24hrs'
    r = requests.get(link)
    soup = BeautifulSoup(r.text, "lxml")
    regex = re.findall(r'"cve": "CVE-\d{4}-\d{4,8}"', soup.get_text())
    top_cve_list = []
    for item in regex:
        top_cve_list.append(re.search(r'CVE-\d{4}-\d{4,8}', item).group())

    return top_cve_list

top_cve_list = get_top_cve_list()  # Получение топа cve с cvetrends.com
print(top_cve_list)

# check https://github.com/nu11secur1ty/ ----------------------------------------------------------------------------- 
def get_exploit_info(cve):
    link = 'https://github.com/nu11secur1ty/CVE-mitre'
    link_2 = 'https://github.com/nu11secur1ty/CVE-mitre/tree/main/2022'
    default_link = ''
    poc_cve_list = []
    r = requests.get(link)
    soup = BeautifulSoup(r.text, "html.parser")
    for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"):
        regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text)
        if regex:
            poc_cve_list.append(str(regex[0]))

    r = requests.get(link_2)
    soup = BeautifulSoup(r.text, "html.parser")
    for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"):
        regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text)
        if regex:
            poc_cve_list.append(str(regex[0]))

    for item in poc_cve_list:
        if cve == item:
            default_link = f'https://github.com/nu11secur1ty/CVE-mitre/tree/main/{cve}'
    return default_link


# check https://github.com/trickest/cve/ ----------------------------------------------------------------------------- 
def get_exploit_info_2(cve):
    # print('get_exploit_info_2') # DEBUG
    year = cve.split('-')[1]
    link = f'https://github.com/trickest/cve/tree/main/{year}'
    r = requests.get(link)
    soup = BeautifulSoup(r.text, "html.parser")
    default_link = ''
    for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"):
        if f'{cve}.md' == cve_id.text:
            default_link = f'**trickest/cve** - https://github.com/trickest/cve/tree/main/{year}/{cve}.md'
            break
    return default_link


exploit_links = []
for t in r.cve.references.reference_data:
    links.append(t.url)
    if 'Exploit' in t.tags:
        exploit_links.append(t.url)
if get_exploit_info(cve):  # check https://github.com/nu11secur1ty/
    exploit_links.append(get_exploit_info(cve))
if get_exploit_info_2(cve):  # check https://github.com/trickest/cve/
    exploit_links.append(get_exploit_info_2(cve))

# parse mitre.org----------------------------------------------------------------------------------------------------- 
def get_mitigations(tactic_id):
    # print("get_mitigations") # DEBUG
    tactic_url = f'https://attack.mitre.org/techniques/{tactic_id}'
    r_get_tactic_info = requests.get(tactic_url)
    s_get_tactic_info = BeautifulSoup(r_get_tactic_info.text, 'html.parser')

    #  0 - Procedure Examples
    #  1 - Mitigations
    #  2 - Detection
    try:
        mitigations = s_get_tactic_info.find_all("table", class_="table table-bordered table-alternate mt-2")[1].text
        buff_string_1 = str(mitigations).replace("ID", "").replace("Mitigation", "").replace("Description", "").replace(
            "\n\n", "")
        buff_list = buff_string_1.split('\n')
        buff_list.pop(0)  # Remove empty element, index - 0
        return_list = []
        for count in range(int(len(buff_list) / 3)):
            id_mit = buff_list[count * 3].rstrip().lstrip()
            name_mit = buff_list[count * 3 + 1].rstrip().lstrip()
            desc_mit = buff_list[count * 3 + 2].rstrip().lstrip()
            return_list.append(f'[{id_mit} - {name_mit}](https://attack.mitre.org/mitigations/{id_mit}) - {desc_mit}')
        return return_list
    except:
        pass


# parse alienvault.com------------------------------------------------------------------------------------------------ 
def get_ttp(cve):
    # print('get_ttp') # DEBUG
    headers_alienvault = {'X-OTX-API-KEY': API_KEY_ALIENVAULT}
    url = f'https://otx.alienvault.com/api/v1/indicators/cve/{cve}'
    pattern_default_answer = '{"detail": "endpoint not found"}'
    r_get_cve_info = requests.get(url, headers=headers_alienvault)
    s_get_cve_info = BeautifulSoup(r_get_cve_info.text, 'lxml')
    tactic_list = []
    if str(s_get_cve_info.p.contents).replace("['", "").replace("']", "") == pattern_default_answer:
        print(f'no information for {cve}')  # DEBUG
    else:
        content = json.loads(str(s_get_cve_info.p).replace("<p>", "").replace("</p>", ""))
        count = int(content['pulse_info']['count'])
        subs_list = []
        for i in range(count):
            subs_list.append(content['pulse_info']['pulses'][i]['subscriber_count'])
        tactic_list = []
        for i, item in enumerate(subs_list):
            max_test = max(subs_list)
            if content['pulse_info']['pulses'][i]['attack_ids']:
                for j in range(len(content['pulse_info']['pulses'][i]['attack_ids'])):
                    tactic_list.append(content['pulse_info']['pulses'][i]['attack_ids'][j]['id'])
                break
            else:
                subs_list.remove(max_test)
        else:
            return f'NO TTP'
    tactic_list = list(set(tactic_list))  # replace replications
    technique_list = []
    for i, item in enumerate(tactic_list):
        if len(str(item)) > 6:
            technique_list.append(str(item).replace(".", "/"))

    result_list = []
    if technique_list:
        for tactic_id in technique_list:
            if get_mitigations(tactic_id):
                for item in get_mitigations(tactic_id):
                    result_list.append(item)
        return list(set(result_list))
    else:
        for tactic_id in tactic_list:
            if get_mitigations(tactic_id):
                for item in get_mitigations(tactic_id):
                    result_list.append(item)
        return list(set(result_list))

def get_data_file():
    data_link = "https://epss.cyentia.com/epss_scores-current.csv.gz"
    response = requests.get(data_link)
    with open(buffer, 'wb') as f:
        f.write(response.content)
    with gzip.open(buffer, 'rb') as f_in:
        with open(filename, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)


def info_data(filename):
    with open(filename) as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        my_file = open(fileout, "w")
        for row in csv_reader:
            try:
                epss = str(round(float(str(100 / (1 / float(row[1])))[0:5]), 2))
                data = str(row[0] + ',' + epss + '\n')
                my_file.write(data)
            except:
                pass
        my_file.close()

filename = f'{PATH}/DATA.csv'
fileout = f'{PATH}/DATA1.csv'
buffer = f'{PATH}/tar.csv.gz'
get_data_file()
info_data(filename)
cve_line = []
procent_line = []
with open(fileout) as csv_file:
    csv_reader = csv.reader(csv_file, delimiter=',')
    for row in csv_reader:
        cve_line.append(row[0])
        procent_line.append(row[1])

def get_kb(cve):

    msrc_url = f"https://api.msrc.microsoft.com/cvrf/v2.0/Updates('{cve}')"
    get_cvrf_link = requests.get(msrc_url, verify=False)
    id_for_cvrf = re.search(r'\d{4}-\w{3}', get_cvrf_link.text)
    cvrf_url = f'https://api.msrc.microsoft.com/cvrf/v2.0/document/{id_for_cvrf[0]}'
    get_info = requests.get(cvrf_url, verify=False)
    soup = BeautifulSoup(get_info.text, "lxml")

    parse_list = []
    buff = ''
    for item in soup.text:
        if item == '\n':
            parse_list.append(buff)
            buff = ''
        else:
            buff += item

    parse_string = ''
    for j, item in enumerate(parse_list):
        regex = re.findall(cve, parse_list[j])
        if regex:
            parse_string = parse_list[j]

    kb_list = re.findall(r'KB\d{7}', parse_string)
    not_remove_list_of_kb = []
    for kb in kb_list:
        if kb not in not_remove_list_of_kb:
            not_remove_list_of_kb.append(kb)

    link_list = []
    for kb in not_remove_list_of_kb:
        kb_url = f'https://catalog.update.microsoft.com/v7/site/Search.aspx?q={kb}'
        test = requests.get(kb_url, verify=False)
        if test.status_code == 200:
            url_get_product = f'https://www.catalog.update.microsoft.com/Search.aspx?q={kb}'
            get_product = requests.get(url_get_product, verify=False)
            soup_get_product = BeautifulSoup(get_product.text, "lxml")
            product_buff = ''
            for item in soup_get_product.find_all('a', class_='contentTextItemSpacerNoBreakLink'):
                product_buff = item.text
            product = product_buff.strip()
            # Output: Windows 10 Version 1809 for x86-based Systems
            #link_list.append(f'[{kb}]({kb_url}) - {(product.partition("for")[2])[:-12]}')
            if product:
                # Output: 2022-01 Cumulative Update for Windows 10 Version 1809 for x86-based Systems (KB5009557)
                link_list.append(f'[{kb}]({kb_url}) - {product[:-12]}')

    return link_list