diff --git a/config/templates/areaLabelAddition.md b/config/templates/areaLabelAddition.md
deleted file mode 100644
index c5e07af..0000000
--- a/config/templates/areaLabelAddition.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Hello {teams} members, this {payload} was labeled with the {refs} {labels}, so you may want to check it out!
-
-<!-- areaLabelAddition -->
\ No newline at end of file
diff --git a/src/events/responses/areaLabel.js b/src/events/responses/areaLabel.js
index dea98e4..fc1e1c3 100644
--- a/src/events/responses/areaLabel.js
+++ b/src/events/responses/areaLabel.js
@@ -1,5 +1,3 @@
-const referenced = [];
-
 exports.run = async function(issue, repo, label) {
   const areaLabel = label.name;
   const number = issue.number;
@@ -17,43 +15,20 @@ exports.run = async function(issue, repo, label) {
   const uniqueTeams = this.util.deduplicate(labelTeams);
 
   const areaTeams = `@${repoOwner}/${uniqueTeams.join(`, @${repoOwner}/`)}`;
-  const references = issueAreaLabels.join("\", \"");
-
-  const payload = issue.pull_request ? "pull request" : "issue";
-  const labelSize = labelTeams.length === 1 ? "label" : "labels";
-  const template = this.templates.get("areaLabelAddition");
-
-  const comment = template.format({
-    teams: areaTeams, refs: `"${references}"`, labels: labelSize,
-    payload: payload
-  });
-
-  const comments = await template.getComments({
-    owner: repoOwner, repo: repoName, number: number
-  });
-
-  const tag = `${repoOwner}/${repoName}#${number}`;
-
-  if (comments.length) {
-    const id = comments[0].id;
-    if (issueAreaLabels.length) {
-      this.issues.editComment({
-        owner: repoOwner, repo: repoName, comment_id: id, body: comment
-      });
-    } else {
-      this.issues.deleteComment({
-        owner: repoOwner, repo: repoName, comment_id: id
-      });
-    }
-  } else if (!referenced.includes(tag) && issueAreaLabels.length) {
-    this.issues.createComment({
-      owner: repoOwner, repo: repoName, number: number, body: comment
-    });
-
-    // Ignore labels added in bulk
-    referenced.push(tag);
-    setTimeout(() => {
-      referenced.splice(referenced.indexOf(tag), 1);
-    }, 1000);
+
+  const prefix = "CC by @zulipbot: ";
+
+  const updatedIssue = {
+    owner: repoOwner, repo: repoName, issue_number: number, body: issue.body
+  };
+
+  const pattern = new RegExp(`${prefix}.+$`, "mg");
+
+  if (issue.description.includes(prefix)) {
+    issue.body = issue.body.replace(pattern, `${prefix}${areaTeams}`);
+  } else {
+    issue.body += `\n\n${prefix}${areaTeams}`;
   }
+
+  this.issues.update(updatedIssue);
 };