Streamlit食谱爬虫应用报错WebsiteNotImplementedError求助

问题详情 #

本地运行main.py脚本时，create_df()函数可正常接收用户提供的食谱URL列表（名为"recipes"），遍历每个URL并返回最终标记为res的DataFrame。但在Streamlit环境中运行时，始终抛出以下错误：

recipe_scrapers._exceptions.WebsiteNotImplementedError: recipe-scrapers exception: Website (h) not supported.

我的需求是为用户生成所提供食谱URL中所有食材的CSV文件，用于制作购物清单。

create_df()函数代码 #

def create_df(recipes):
    """
    Description:
    Creates one df with all recipes and their ingredients

    Arguments:
    * recipes: list of recipe URLs provided by user

    Comments:
    Note that ingredients with qualitative amounts e.g., "scheutje melk", "snufje zout" have been ommitted from the ingredient list
    """
    df_list = []

    for recipe in recipes:
        scraper = scrape_me(recipe)
        recipe_details = replace_measurement_symbols(scraper.ingredients())

        recipe_name = recipe.split("https://www.hellofresh.nl/recipes/", 1)[1]
        recipe_name = recipe_name.rsplit('-', 1)[0]
        print("Processing data for "+ recipe_name +" recipe.")

        for ingredient in recipe_details:
            try:
                df_temp = pd.DataFrame(columns=['Ingredients', 'Measurement'])
                df_temp[str(recipe_name)] = recipe_name

                ing_1 = ingredient.split("2 * ", 1)[1]
                ing_1 = ing_1.split(" ", 2)

                item = ing_1[2]
                measurement = ing_1[1]
                quantity = float(ing_1[0]) * 2

                df_temp.loc[len(df_temp)] = [item, measurement, quantity]
                df_list.append(df_temp)
            except (ValueError, IndexError) as e:
                pass

        df = pd.concat(df_list)

    print("Renaming duplicate ingredients e.g., Kruimige aardappelen, Voorgekookte halve kriel met schil -> Aardappelen")
    ingredient_dict = {
                    'Aardappelen': ('Dunne frieten', 'Half kruimige aardappelen', 'Voorgekookte halve kriel met schil',
                                    'Kruimige aardappelen', 'Roodschillige aardappelen', 'Opperdoezer Ronde aardappelen'),
                    'Ui': ('Rode ui'),
                    'Kipfilet': ('Kipfilet met tuinkruiden en knoflook'),
                    'Kipworst': ('Gekruide kipworst'),
                    'Kipgehakt': ('Gemengd gekruid gehakt', 'Kipgehakt met Mexicaanse kruiden', 'Half-om-halfgehakt met Italiaanse kruiden',
                                  'Kipgehakt met tuinkruiden'),
                    'Kipshoarma': ('Kalkoenshoarma')
                    }

    reverse_label_ing = {x:k for k,v in ingredient_dict.items() for x in v}
    df["Ingredients"].replace(reverse_label_ing, inplace=True)

    print("Assigning ingredient categories")
    category_dict = {
                    'brood': ('Biologisch wit rozenbroodje', 'Bladerdeeg', 'Briochebroodje', 'Wit platbrood'),
                    'granen': ('Basmatirijst', 'Bulgur', 'Casarecce', 'Cashewstukjes',
                               'Gesneden snijbonen', 'Jasmijnrijst', 'Linzen', 'Maïs in blik',
                               'Parelcouscous', 'Penne', 'Rigatoni', 'Rode kidneybonen',
                               'Spaghetti', 'Witte tortilla'),
                    'groenten': ('Aardappelen', 'Aubergine', 'Bosui', 'Broccoli',
                                 'Champignons', 'Citroen', 'Gele wortel', 'Gesneden rodekool',
                                 'Groene paprika', 'Groentemix van paprika, prei, gele wortel en courgette',
                                 'IJsbergsla', 'Kumato tomaat', 'Limoen', 'Little gem',
                                 'Paprika', 'Portobello', 'Prei', 'Pruimtomaat',
                                 'Radicchio en ijsbergsla', 'Rode cherrytomaten', 'Rode paprika', 'Rode peper',
                                 'Rode puntpaprika', 'Rode ui', 'Rucola', 'Rucola en veldsla', 'Rucolamelange',
                                 'Semi-gedroogde tomatenmix', 'Sjalot', 'Sperziebonen', 'Spinazie', 'Tomaat',
                                 'Turkse groene peper', 'Veldsla', 'Vers basilicum', 'Verse bieslook',
                                 'Verse bladpeterselie', 'Verse koriander', 'Verse krulpeterselie', 'Wortel', 'Zoete aardappel'),
                    'kruiden': ('Aïoli', 'Bloem', 'Bruine suiker', 'Cranberrychutney', 'Extra vierge olijfolie',
                                'Extra vierge olijfolie met truffelaroma', 'Fles olijfolie', 'Gedroogde laos',
                                'Gedroogde oregano', 'Gemalen kaneel', 'Gemalen komijnzaad', 'Gemalen korianderzaad',
                                'Gemalen kurkuma', 'Gerookt paprikapoeder', 'Groene currykruiden', 'Groentebouillon',
                                'Groentebouillonblokje', 'Honing', 'Italiaanse kruiden', 'Kippenbouillonblokje', 'Knoflookteen',
                                'Kokosmelk', 'Koreaanse kruidenmix', 'Mayonaise', 'Mexicaanse kruiden', 'Midden-Oosterse kruidenmix',
                                'Mosterd', 'Nootmuskaat', 'Olijfolie', 'Panko paneermeel', 'Paprikapoeder', 'Passata',
                                'Pikante uienchutney', 'Runderbouillonblokje', 'Sambal', 'Sesamzaad', 'Siciliaanse kruidenmix',
                                'Sojasaus', 'Suiker', 'Sumak', 'Surinaamse kruiden', 'Tomatenblokjes', 'Tomatenblokjes met ui',
                                'Truffeltapenade', 'Ui', 'Verse gember', 'Visbouillon', 'Witte balsamicoazijn', 'Wittewijnazijn',
                                'Zonnebloemolie', 'Zwarte balsamicoazijn'),
                    'vlees': ('Gekruide runderburger', 'Half-om-half gehaktballetjes met Spaanse kruiden', 'Kipfilethaasjes', 'Kipfiletstukjes',
                              'Kipgehaktballetjes met Italiaanse kruiden', 'Kippendijreepjes', 'Kipshoarma', 'Kipworst', 'Spekblokjes',
                              'Vegetarische döner kebab', 'Vegetarische kaasschnitzel', 'Vegetarische schnitzel'),
                    'zuivel': ('Ei', 'Geraspte belegen kaas', 'Geraspte cheddar', 'Geraspte grana padano', 'Geraspte oude kaas',
                               'Geraspte pecorino', 'Karnemelk', 'Kruidenroomkaas', 'Labne', 'Melk', 'Mozzarella',
                               'Parmigiano reggiano', 'Roomboter', 'Slagroom', 'Volle yoghurt')
                    }

    reverse_label_cat = {x:k for k,v in category_dict.items() for x in v}
    df["Category"] = df["Ingredients"].map(reverse_label_cat)
    col = "Category"
    first_col = df.pop(col)
    df.insert(0, col, first_col)
    df = df.sort_values(['Category', 'Ingredients'], ascending = [True, True])

    print("Merging ingredients by row across all recipe columns using justify()")
    gp_cols = ['Ingredients', 'Measurement']
    oth_cols = df.columns.difference(gp_cols)

    arr = np.vstack(df.groupby(gp_cols, sort=False, dropna=False).apply(lambda gp: justify(gp.to_numpy(), invalid_val=np.NaN, axis=0, side='up')))

    # Reconstruct DataFrame
    # Remove entirely NaN rows based on the non-grouping columns
    res = (pd.DataFrame(arr, columns=df.columns)
             .dropna(how='all', subset=oth_cols, axis=0))

    res = res.fillna(0)
    res['Total'] = res.drop(['Ingredients', 'Measurement'], axis=1).sum(axis=1)
    res=res[res['Total'] !=0] #To drop rows that are being duplicated with 0 for some reason; will check later

    print("Processing complete!")

    return res

内容的提问来源于stack exchange，提问作者June Smith