Add support for HF summarization endpoint in the websearch (huggingfa…

…ce#319)

* Add support for HF endpoint for summary

* add fail-safe for summarization

src/lib/server/websearch/summarizeWeb.ts

@@ -1,7 +1,28 @@
+import { HF_ACCESS_TOKEN } from "$env/static/private";
+import { HfInference } from "@huggingface/inference";
 import { generateFromDefaultEndpoint } from "../generateFromDefaultEndpoint";
 import type { BackendModel } from "../models";
 
 export async function summarizeWeb(content: string, query: string, model: BackendModel) {
+ // if HF_ACCESS_TOKEN is set, we use a HF dedicated endpoint for summarization
+ try {
+ if (HF_ACCESS_TOKEN) {
+ const summary = (
+ await new HfInference(HF_ACCESS_TOKEN).summarization({
+ model: "facebook/bart-large-cnn",
+ inputs: content,
+ parameters: {
+ max_length: 512,
+ },
+ })
+ ).summary_text;
+ return summary;
+ }
+ } catch (e) {
+ console.log(e);
+ }
+
+ // else we use the LLM to generate a summary
  const summaryPrompt =
  model.userMessageToken +
  content