Merge pull request #6346 from ForumMagnum/languageModelIntegration

Auto-apply core tags

.gitignore

@@ -91,3 +91,5 @@ myLocalDatabase/*
 schema/schema_to_accept.sql
 
 .branchdbcache
+
+ml

build.js

@@ -189,7 +189,8 @@ build({
     "mathjax", "mathjax-node", "mathjax-node-page", "jsdom", "@sentry/node", "node-fetch", "later", "turndown",
     "apollo-server", "apollo-server-express", "graphql", "csso", "io-ts", "fp-ts",
     "bcrypt", "node-pre-gyp", "intercom-client", "node:*",
-    "fsevents", "chokidar", "auth0", "dd-trace", "pg-formatter"
+    "fsevents", "chokidar", "auth0", "dd-trace", "pg-formatter",
+    "gpt-3-encoder",
   ],
 })
 

package.json

@@ -198,6 +198,7 @@
     "express-session": "^1.17.1",
     "feedparser-promised": "^1.2.2",
     "fp-ts": "^2.13.1",
+    "gpt-3-encoder": "^1.1.3",
     "graphql": "^14.5.8",
     "graphql-date": "^1.0.2",
     "graphql-server-express": "^0.6.0",
@@ -236,6 +237,7 @@
     "mongodb": "^3.6.3",
     "mz": "^2.7.0",
     "nodemailer": "^6.6.1",
+    "openai": "^3.1.0",
     "papaparse": "^4.6.2",
     "passport": "^0.5.2",
     "passport-auth0": "^1.4.0",

packages/lesswrong/components/languageModels/PostSummaryAction.tsx

@@ -0,0 +1,61 @@
+import React from 'react';
+import { registerComponent, Components } from '../../lib/vulcan-lib';
+import { useDialog } from '../common/withDialog';
+import { useSingle } from '../../lib/crud/withSingle';
+import DialogTitle from '@material-ui/core/DialogTitle';
+import DialogContent from '@material-ui/core/DialogContent';
+import MenuItem from '@material-ui/core/MenuItem';
+
+const styles = (theme: ThemeType) => ({
+})
+
+const PostSummaryAction = ({post, closeMenu, classes}: {
+  post: PostsList,
+  closeMenu?: ()=>void,
+  classes: ClassesType,
+}) => {
+  const { openDialog } = useDialog();
+
+  const showPostSummary = () => {
+    closeMenu?.();
+    openDialog({
+      componentName: "PostSummaryDialog",
+      componentProps: { post },
+    });
+  }
+
+  return <MenuItem onClick={showPostSummary}>
+    Summarize
+  </MenuItem>
+}
+
+const PostSummaryDialog = ({post, onClose, classes}: {
+  post: PostsList,
+  onClose?: ()=>void,
+  classes: ClassesType,
+}) => {
+  const { Loading, LWDialog } = Components;
+  const { document: postWithSummary, loading } = useSingle({
+    collectionName: "Posts",
+    fragmentName: "PostWithGeneratedSummary",
+    documentId: post._id,
+  });
+
+  return <LWDialog open={true} onClose={onClose}>
+    <DialogTitle>{post.title}</DialogTitle>
+    <DialogContent>
+      {loading && <Loading/>}
+      {postWithSummary && postWithSummary.languageModelSummary}
+    </DialogContent>
+  </LWDialog>
+}
+
+const PostSummaryActionComponent = registerComponent('PostSummaryAction', PostSummaryAction, {styles});
+const PostSummaryDialogComponent = registerComponent('PostSummaryDialog', PostSummaryDialog, {styles});
+
+declare global {
+  interface ComponentTypes {
+    PostSummaryAction: typeof PostSummaryActionComponent
+    PostSummaryDialog: typeof PostSummaryDialogComponent
+  }
+}

packages/lesswrong/components/posts/PostsPage/PostActions.tsx

@@ -22,6 +22,7 @@ import { subscriptionTypes } from '../../../lib/collections/subscriptions/schema
 import { useDialog } from '../../common/withDialog';
 import { forumTypeSetting, taggingNamePluralCapitalSetting } from '../../../lib/instanceSettings';
 import { forumSelect } from '../../../lib/forumTypeUtils';
+import { userHasAutosummarize } from '../../../lib/betas';
 
 // We use a context here vs. passing in a boolean prop because we'd need to pass through ~4 layers of hierarchy
 export const AllowHidingFrontPagePostsContext = React.createContext<boolean>(false)
@@ -282,6 +283,10 @@ const PostActions = ({post, closeMenu, classes}: {
             Edit {taggingNamePluralCapitalSetting.get()}
           </MenuItem>
         </div>
+
+        {userHasAutosummarize(currentUser)
+          && <Components.PostSummaryAction closeMenu={closeMenu} post={post}/>}
+
         { isRead
           ? <div onClick={handleMarkAsUnread}>
               <MenuItem>

packages/lesswrong/components/tagging/FooterTag.tsx

@@ -112,7 +112,7 @@ const FooterTag = ({tagRel, tag, hideScore=false, classes, smallText, popperCard
     tagName: tag.name,
     tagSlug: tag.slug
   });
-  const { PopperCard, TagRelCard, TopTagIcon, TagPreview } = Components
+  const { PopperCard, TagRelCard, TopTagIcon } = Components
 
   const sectionContextMaybe = isTopTag ? {pageSectionContext: 'topTag'} : {}
 

packages/lesswrong/components/tagging/TagPreview.tsx

@@ -30,16 +30,21 @@ const styles = (theme: ThemeType): JssStyles => ({
       width: "100%",
     }
   },
-  footerCount: {
+  footer: {
     borderTop: theme.palette.border.extraFaint,
     paddingTop: 6,
-    textAlign: "right",
+    display: "flex",
     ...theme.typography.smallFont,
     ...theme.typography.commentStyle,
     color: theme.palette.lwTertiary.main,
     marginTop: 6,
     marginBottom: 2
   },
+  autoApplied: {
+    flexGrow: 1,
+  },
+  footerCount: {
+  },
   posts: {
     marginTop: 10,
     paddingTop: 8,
@@ -60,9 +65,10 @@ export type TagPreviewProps = {
   showCount?: boolean,
   showRelatedTags?: boolean,
   postCount?: number,
+  autoApplied?: boolean,
 }
 
-const TagPreview = ({tag, loading, classes, showCount=true, showRelatedTags=true, postCount=6}: TagPreviewProps) => {
+const TagPreview = ({tag, loading, classes, showCount=true, showRelatedTags=true, postCount=6, autoApplied=false}: TagPreviewProps) => {
   const { TagPreviewDescription, TagSmallPostLink, Loading } = Components;
   const { results } = useMulti({
     skip: !(tag?._id),
@@ -78,6 +84,8 @@ const TagPreview = ({tag, loading, classes, showCount=true, showRelatedTags=true
     return null
   }
 
+  const hasFooter = (showCount || autoApplied);
+
   return (<div className={classes.card}>
     {loading && <Loading />}
     {tag && <>
@@ -94,8 +102,13 @@ const TagPreview = ({tag, loading, classes, showCount=true, showRelatedTags=true
         {results ? <div className={classes.posts}>
           {results.map((post,i) => post && <TagSmallPostLink key={post._id} post={post} widerSpacing={postCount > 3} />)}
         </div> : <Loading /> }
-        {showCount && <div className={classes.footerCount}>
-          <Link to={tagGetUrl(tag)}>View all {tag.postCount} posts</Link>
+        {hasFooter && <div className={classes.footer}>
+          {autoApplied && <span className={classes.autoApplied}>
+            Tag was auto-applied
+          </span>}
+          {showCount && <span className={classes.footerCount}>
+            <Link to={tagGetUrl(tag)}>View all {tag.postCount} posts</Link>
+          </span>}
         </div>}
       </>}
     </>}

packages/lesswrong/components/tagging/TagRelCard.tsx

@@ -86,7 +86,7 @@ const TagRelCard = ({tagRel, classes, relevance=true}: {
       </span>}
       {voteProps.baseScore <= 0 && <span className={classes.removed}>Removed (refresh page)</span>}
     </div>
-    {tagRel.tag && <TagPreview tag={tagRel.tag}/>}
+    {tagRel.tag && <TagPreview tag={tagRel.tag} autoApplied={tagRel.autoApplied}/>}
   </div>
 }
 

packages/lesswrong/lib/betas.ts

@@ -32,6 +32,8 @@ export const userCanUseSharing = (user: UsersCurrent|DbUser|null): boolean => mo
 export const userHasNewTagSubscriptions =  isEAForum ? shippedFeature : disabled
 export const userHasDefaultProfilePhotos = disabled
 
+export const userHasAutosummarize = adminOnly
+
 export const userHasThemePicker = isEAForum ? adminOnly : shippedFeature;
 
 export const userHasSideComments = isEAForum ? optInOnly : shippedFeature;

packages/lesswrong/lib/collections/posts/fragments.ts

@@ -577,3 +577,10 @@ registerFragment(`
     sideComments
   }
 `);
+
+registerFragment(`
+  fragment PostWithGeneratedSummary on Post {
+    _id
+    languageModelSummary
+  }
+`);

packages/lesswrong/lib/collections/posts/schema.tsx

@@ -2345,6 +2345,14 @@ const schema: SchemaType<DbPost> = {
     type: Object,
     foreignKey: 'Comments',
   },
+
+  languageModelSummary: {
+    type: String,
+    optional: true,
+    hidden: true,
+    canRead: ['admins'],
+    // Implementation in postSummaryResolver.ts
+  },
 };
 
 /* subforum-related fields */

packages/lesswrong/lib/collections/tagRels/collection.ts

@@ -59,6 +59,13 @@ const schema: SchemaType<DbTagRel> = {
       return currentUser ? !(await userCanVoteOnTag(currentUser, document.tagId)).fail : true;
     },
   }),
+
+  autoApplied: {
+    type: Boolean,
+    viewableBy: ['guests'],
+    optional: true, hidden: true,
+    // Implementation in tagResolvers.ts
+  },
 };
 
 export const TagRels: TagRelsCollection = createCollection({

packages/lesswrong/lib/collections/tagRels/fragments.ts

@@ -11,6 +11,7 @@ registerFragment(`
     userId
     tagId
     postId
+    autoApplied
   }
 `);
 

packages/lesswrong/lib/collections/tags/fragments.ts

@@ -280,6 +280,10 @@ registerFragment(`
     }
     tagFlagsIds
     postsDefaultSortOrder
+    
+    autoTagModel
+    autoTagPrompt
+    
     description {
       ...RevisionEdit
     }

packages/lesswrong/lib/collections/tags/schema.ts

@@ -559,6 +559,30 @@ const schema: SchemaType<DbTag> = {
     type: Object,
     foreignKey: 'Tags',
   },
+
+  autoTagModel: {
+    type: String,
+    label: "Auto-tag classifier model ID",
+    optional: true,
+    viewableBy: ['admins'],
+    editableBy: ['admins'],
+    insertableBy: ['admins'],
+    group: formGroups.advancedOptions,
+    nullable: true,
+    ...schemaDefaultValue(""),
+  },
+
+  autoTagPrompt: {
+    type: String,
+    label: "Auto-tag classifier prompt string",
+    optional: true,
+    viewableBy: ['admins'],
+    editableBy: ['admins'],
+    insertableBy: ['admins'],
+    group: formGroups.advancedOptions,
+    nullable: true,
+    ...schemaDefaultValue(""),
+  },
 }
 
 export default schema;

packages/lesswrong/lib/components.ts

@@ -690,6 +690,8 @@ importComponent("PrefixedInput", () => require('../components/form-components/Pr
 importComponent("PodcastEpisodeInput", () => require('../components/form-components/PodcastEpisodeInput'));
 importComponent("ThemeSelect", () => require('../components/form-components/ThemeSelect'));
 
+importComponent(["PostSummaryAction","PostSummaryDialog"], () => require('../components/languageModels/PostSummaryAction'));
+
 importComponent(["CommentOnSelectionPageWrapper","SelectedTextToolbar","CommentOnSelectionContentWrapper"], () => require('../components/comments/CommentOnSelection'));
 importComponent("PopupCommentEditor", () => require('../components/comments/PopupCommentEditor'));
 

packages/lesswrong/lib/generated/databaseTypes.d.ts

@@ -878,6 +878,8 @@ interface DbTag extends DbObject {
   isSubforum: boolean
   subforumModeratorIds: Array<string>
   parentTagId: string
+  autoTagModel: string | null
+  autoTagPrompt: string | null
   createdAt: Date
   legacyData: any /*{"definitions":[{"blackbox":true}]}*/
   description: EditableFieldContents

packages/lesswrong/lib/generated/fragmentTypes.d.ts

@@ -462,13 +462,16 @@ interface TagsDefaultFragment { // fragment on Tags
   readonly isSubforum: boolean,
   readonly subforumModeratorIds: Array<string>,
   readonly parentTagId: string,
+  readonly autoTagModel: string | null,
+  readonly autoTagPrompt: string | null,
 }
 
 interface TagRelsDefaultFragment { // fragment on TagRels
   readonly tagId: string,
   readonly postId: string,
   readonly deleted: boolean,
   readonly userId: string,
+  readonly autoApplied: boolean,
 }
 
 interface BooksDefaultFragment { // fragment on Books
@@ -655,6 +658,7 @@ interface PostsDefaultFragment { // fragment on Posts
   readonly moderationStyle: string,
   readonly hideCommentKarma: boolean,
   readonly commentCount: number,
+  readonly languageModelSummary: string,
   readonly subforumTagId: string,
   readonly af: boolean,
   readonly afDate: Date,
@@ -1183,6 +1187,11 @@ interface PostSideComments { // fragment on Posts
   readonly sideComments: any,
 }
 
+interface PostWithGeneratedSummary { // fragment on Posts
+  readonly _id: string,
+  readonly languageModelSummary: string,
+}
+
 interface CommentsList { // fragment on Comments
   readonly _id: string,
   readonly postId: string,
@@ -1858,6 +1867,7 @@ interface TagRelBasicInfo { // fragment on TagRels
   readonly userId: string,
   readonly tagId: string,
   readonly postId: string,
+  readonly autoApplied: boolean,
 }
 
 interface TagRelFragment extends TagRelBasicInfo { // fragment on TagRels
@@ -2122,6 +2132,8 @@ interface TagEditFragment extends TagDetailsFragment { // fragment on Tags
   readonly parentTag: TagEditFragment_parentTag|null,
   readonly tagFlagsIds: Array<string>,
   readonly postsDefaultSortOrder: string,
+  readonly autoTagModel: string | null,
+  readonly autoTagPrompt: string | null,
   readonly description: RevisionEdit|null,
   readonly subforumWelcomeText: RevisionEdit|null,
   readonly moderationGuidelines: RevisionEdit|null,
@@ -2904,6 +2916,7 @@ interface FragmentTypes {
   WithVotePost: WithVotePost
   HighlightWithHash: HighlightWithHash
   PostSideComments: PostSideComments
+  PostWithGeneratedSummary: PostWithGeneratedSummary
   CommentsList: CommentsList
   ShortformComments: ShortformComments
   CommentWithRepliesFragment: CommentWithRepliesFragment
@@ -3081,6 +3094,7 @@ interface CollectionNamesByFragmentName {
   WithVotePost: "Posts"
   HighlightWithHash: "Posts"
   PostSideComments: "Posts"
+  PostWithGeneratedSummary: "Posts"
   CommentsList: "Comments"
   ShortformComments: "Comments"
   CommentWithRepliesFragment: "Comments"

packages/lesswrong/server.ts

@@ -54,6 +54,7 @@ import './server/scripts/validateMakeEditableDenormalization';
 import './server/scripts/mergeAccounts';
 import "./server/scripts/testPostDescription";
 import "./server/scripts/importEAGUserInterests";
+import "./server/scripts/languageModels/generateTaggingPostSets";
 import './server/manualMigrations';
 import './server/manualMigrations/migrationsDashboardGraphql';
 
@@ -146,8 +147,11 @@ import './server/fmCrosspost/routes';
 
 import './server/spotlightCron';
 
-import './server/codegen/generateTypes';
+import "./server/languageModels/autoTagCallbacks";
+import './server/languageModels/languageModelIntegration';
+import './server/languageModels/postSummaryResolver';
 
+import './server/codegen/generateTypes';
 import './server/styleGeneration';
 
 // Algolia Search Integration