par_dict = {"id":45,"page":1,"params":{}}

headers = {
     "Accept": "application/json, text/javascript, */*; q=0.01",
     "Accept-Encoding": "gzip, deflate",
     "Accept-Language": "en-US,en;q=0.9,ru-RU;q=0.8,ru;q=0.7,la;q=0.6",
     "Connection": "keep-alive",
     "Content-Length": "100",
     "Content-Type": "application/json; charset=UTF-8",
     "Cookie": "_ym_uid=15260890961045257376; _ym_isad=1; _ym_visorc_31062401=w; _vagrant_session=V2Qwb3V0aWc4K3NDVW1KVkdxQ0xzZjEyL3JIM2JuSlBGeFJyTEJhUHVXZkZEWnArdUJ6eEVXRFJBcmhrQVh0dWtpVG9iQ0g0UDJXNmtIR0lYUkhJaDhEMkVscUdvSzZhOUFJSlVDSUlqTTlzdXJEY0dpY1Jsa1Q5SzRSb01VdDRDWGVuaGtzSStaazYyYmdnOWFxcWRvSkJ2RFhxM0hadHdyT3hUNEgrY0hFPS0tUTNrUmY2eUx0bXVQdDlYbTkxZzFhQT09--346f51dc5bbd378a982946b4dc5b95d5b5e3038a",
     "Host": "vo.graduate.edu.ru",
     "Origin": "http://vo.graduate.edu.ru",
     "Referer": "http://vo.graduate.edu.ru/registry",
     "Save-Data": "on",
     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
     "X-CSRF-Token": "ZxpQgT3/JsEF/evo5FFLH2niFgeDSrdl8BLQ798xzHiG3umyqLy/Yt61aMwx9jd92/2hitrsaSGN5siwVLrGRQ==",
     "X-Requested-With": "XMLHttpRequest"
}

par_json = json.dumps(par_dict)

r = requests.post("http://vo.graduate.edu.ru/graphs/getGraph", data=par_json)

r

r = requests.post("http://vo.graduate.edu.ru/graphs/getGraph", data=par_json, headers=headers)
r

from bs4 import BeautifulSoup

soup = BeautifulSoup("какой-то текст <b class='class_name', id='id_attr'>Полужирный текст</b> <i>И ещё немножко</i>", "lxml")
soup

btag = soup.b
btag

dir(btag)

btag.name

btag.name = "span"
btag

btag["class"]

btag.attrs

btag["id"] = "some_id"
btag

btag.string

btag.string.replace_with("Новый текст")
btag

soup.get_text()

sibling_soup = BeautifulSoup("<a><b class='cl'>text1</b><b class='cl'>text2</b></b></a>", "lxml")
print(sibling_soup.prettify())

sibling_soup.find("b", attrs={"class": "cl"})
sibling_soup.find("b", class_="cl")

all_b = sibling_soup.find_all("b", class_="cl")
all_b

for b in all_b:
    print(b.get_text())

import requests
from pyquery import PyQuery as pq
import json
from json import JSONDecodeError
from tqdm import tqdm_notebook
import pandas as pd

url = "https://eda.ru/recepty?page={}"
res = requests.get(url.format(1))

recipes_subcat_list = []
for recipes_cat in pq(res.text).find(".seo-footer .seo-footer__list"):
    recipes_subcats = pq(recipes_cat).find("li.seo-footer__list-item, li.seo-footer__list-title _empty")
    for recipes_subcat in recipes_subcats:
        d = {
            "title": pq(recipes_subcat).text().split("\xa0\xa0")[0],
            "href": pq(recipes_subcat).find("a").attr("href"),
            "num": int(pq(recipes_subcat).text().split("\xa0\xa0")[1])
        }
        recipes_subcat_list.append(d)

data = {}

for rec_cat in recipes_subcat_list:
    print(rec_cat["title"])
    has_items = True
    page = 1
    
    while has_items:
        res = requests.get("https://eda.ru/{cat}?page={page}".format(cat=rec_cat["href"], page=page)).text
        
        if len(pq(res).find(".recipes-page__recipes .tile-list__horizontal-tile")):
            page += 1
            for recipe in pq(res).find(".recipes-page__recipes .tile-list__horizontal-tile"):
                title = pq(recipe).find(".horizontal-tile__item-title").text()
                href = pq(recipe).find(".horizontal-tile__item-title a").attr("href")
#                 print("Page {}, URL {}".format(page, href), end="\r")
                img_src = pq(recipe).find(".horizontal-tile__preview .lazy-load-container").attr("data-src")
                booked = int(pq(recipe).find(".js-bookmark__counter").text())
                likes = int(pq(recipe).find(".widget-list__like-count").text().split()[0])
                dislikes = int(pq(recipe).find(".widget-list__like-count").text().split()[1])
                time_to_cook = pq(recipe).find(".prep-time").text()
                portions = pq(recipe).find(".js-portions-count-print").text()
                ingredients = []
                for ingredient in pq(recipe).find(".ingredients-list .ingredients-list__content-item"):
                    try:
                        attr = pq(ingredient).attr("data-ingredient-object")
                        ingredients.append(json.loads(attr.replace('""', '"')))
                    except JSONDecodeError as err:
                        print(pq(ingredient).attr("data-ingredient-object"))
                        break
                
                data[href] = {
                    "title": title,
                    "img_src": img_src,
                    "ingredients": ingredients,
                    "booked": booked,
                    "likes": likes,
                    "dislikes": dislikes,
                    "time_to_cook": time_to_cook,
                    "portions": portions,
                    "category": rec_cat["title"]
                }
        else:
            has_items = False

len(data)

for num, d in enumerate(data):
    data[d]["id"] = num

json.dump(data, open("recipes.json", "wt", encoding="utf8"))

data = json.load(open("recipes.json", "rt", encoding="utf8"))

df = pd.DataFrame.from_dict(data, orient="index")

df["img_src"].isnull().sum()

df["category"].head()

df.to_msgpack("recipes.msg")

for url, d in tqdm_notebook(data.items()):
    if d["img_src"]:
        if d["id"] in range(14419, 40000): # для возобновления закачки с какого-либо момента
            url = "https:" + d["img_src"]
            res = requests.get(url)
            path = "imgs/" + str(d["id"]) + "." + d["img_src"].split(".")[-1]
            open(path, 'wb').write(res.content)

cat = {}
for recipes_list in pq(res.text).find(".seo-footer .seo-footer__list"):
    rec_lis = pq(recipes_list).find("li")
    cat_name = pq(rec_lis[0]).text().split("\xa0\xa0")[0]

    for rec_li in pq(rec_lis[1:]):
        subcat_name = pq(rec_li).text().split("\xa0\xa0")[0]
        cat[subcat_name] = cat_name

for num, d in enumerate(data):
    data[d]["sup_category"] = cat[data[d]["category"]]

import requests

access_token = '0b9765dd0b9765dd0b9765dd640bfff4a600b970b9765dd57ca9d0a3a19de66999e2ac0'
api_version = '5.89'
res = requests.get(f'https://api.vk.com/method/users.get?user_ids=1,23,4,3&access_token={access_token}&v={api_version}')
res.json()

res = requests.get(f'https://api.vk.com/method/wall.get?domain=memesocfuck&count=1&access_token={access_token}&v={api_version}')
res.json()