SEO adjustments for KB forums and user-related links (#5320)

* disallow crawling of KB forums via robots.txt
* add rel='ugc nofollow' to KB forum links
* add tests
* add rel='nofollow' to user links
* add doc for SEO notes/policies

docs/index.rst

@@ -31,6 +31,7 @@ Part 2: Developer's Guide
  svelte
  browser_permissions
  zendesk
+ seo
  notes
 
 

docs/seo.rst

@@ -0,0 +1,60 @@
+===
+SEO
+===
+
+This document covers notes and policies related to SEO.
+
+Prefer ``meta`` tag if possible
+=============================
+
+If an entire page should not be indexed, and/or none of its links
+followed, prefer to use the ``<meta name="robots" ...>`` tag by specifying
+something like::
+
+ {% set meta = (('robots', 'noindex'),) %}
+
+or::
+
+ {% set meta = (('robots', 'noindex, nofollow'),) %}
+
+
+within the lowest-level Jinja2 templates of the inheritance chain that
+apply to only the desired pages.
+
+However, if you only want to discourage the crawling of specific links within
+a page, you'll have to add ``rel="nofollow"`` to each of those links within its
+template. For example::
+
+ <a rel="nofollow" href="...">...</a>
+
+
+Breadcrumbs
+===========
+
+If one or more of the breadcrumb links for a page should not be crawled, you
+can add an extra string to those breadcrumb tuples to specify the proper
+attribute to use, for example::
+
+ {% set crumbs = [((profile_url(user), 'rel="nofollow"'), user.username), (None, title)] %}
+
+or::
+
+ {% set crumbs = [(document.get_absolute_url(), document.title), ((url('wiki.discuss.threads', document.slug), 'rel="ugc nofollow"'), _('Discuss'))] %}
+
+KB Forums
+=========
+
+KB forums are user-generated content about KB articles. They are not
+official content, and therefore not meant to be searchable. All links to
+KB forums should be marked with ``rel="ugc nofollow"``.
+
+User Links
+==========
+
+User-related pages are also not meant to be indexed (searchable), and links
+to them should not be crawled, so the base user template
+(``kitsune/users/jinja2/users/base.html``) contains::
+
+ {% set meta = (('robots', 'noindex'),) %}
+
+and all user links on all pages should be marked with ``rel="nofollow"``.

kitsune/community/jinja2/community/index.html

@@ -156,7 +156,7 @@ <h2 class="sumo-page-subheading">{{ _('Localization') }}</h2>
  {% for result in results %}
  <li>
  {% set tooltip = _('{user} - {num} contributions in last 90 days')|f(user=result['user']['display_name'], num=result['count']) %}
- <a href="{{ url('users.profile', result['user']['id']) }}" title="{{ tooltip }}">
+ <a rel="nofollow" href="{{ url('users.profile', result['user']['id']) }}" title="{{ tooltip }}">
  <img src="{{ result['user']['avatar'] }}" alt="{{ tooltip }}" />
  </a>
  </li>

kitsune/community/jinja2/community/search.html

@@ -37,7 +37,7 @@ <h1 class="sumo-page-heading">{{ _('Find your fellow contributors') }}</h1>
  <h2 class="card--title">{{ result['name'] }}</h2>
  <ul class="results-user-details">
  <li>{{ result['username'] }}</li>
- <li><a href="{{ url('users.profile', result['user_id']) }}">{{ _('View Profile') }}</a></li>
+ <li><a rel="nofollow" href="{{ url('users.profile', result['user_id']) }}">{{ _('View Profile') }}</a></li>
  <li><a href="{{ url('messages.new')|urlparams(to=result['username']) }}">{{ _('Private Message') }}</a></li>
  </ul>
  </div>

kitsune/community/jinja2/community/top_contributors.html

@@ -114,7 +114,7 @@ <h2 class="card--title">{{ result['user']['display_name'] }}</h2>
  <ul class="results-user-details">
  <li>{{ result['user']['username'] }}</li>
  <li>{{ _('Contributions in last 90 days: {count}')|f(count=result['count']) }}</li>
- <li><a href="{{ url('users.profile', result['user']['id']) }}">{{ _('View Profile') }}</a></li>
+ <li><a rel="nofollow" href="{{ url('users.profile', result['user']['id']) }}">{{ _('View Profile') }}</a></li>
  <li><a href="{{ url('messages.new')|urlparams(to=result['user']['username']) }}">{{ _('Private Message') }}</a></li>
  {% if result['user']['days_since_last_activity'] is not undefined %}
  <li>

kitsune/flagit/jinja2/flagit/includes/flagged_post.html

@@ -23,10 +23,10 @@ <h3 class="sumo-page-intro">{{ _('Take Action:') }}</h3>
  <a class="sumo-button button-sm" href="{{ object.content_object.get_absolute_url() }}">View</a>
  {% if object.content_type.name == 'KB Forum Post' %}
  {% if user.has_perm('kbforums.change_post') %}
- <a class="sumo-button button-sm edit" href="{{ url('wiki.discuss.edit_post', object.content_object.thread.document.slug, object.content_object.thread.id, object.content_object.id) }}">{{ _('Edit') }}</a>
+ <a class="sumo-button button-sm edit" rel="ugc nofollow" href="{{ url('wiki.discuss.edit_post', object.content_object.thread.document.slug, object.content_object.thread.id, object.content_object.id) }}">{{ _('Edit') }}</a>
  {% endif %}
  {% if user.has_perm('kbforums.delete_post') %}
- <a class="sumo-button button-sm delete" href="{{ url('wiki.discuss.delete_post', object.content_object.thread.document.slug, object.content_object.thread.id, object.content_object.id) }}">{{ _('Delete') }}</a>
+ <a class="sumo-button button-sm delete" rel="ugc nofollow" href="{{ url('wiki.discuss.delete_post', object.content_object.thread.document.slug, object.content_object.thread.id, object.content_object.id) }}">{{ _('Delete') }}</a>
  {% endif %}
  {% endif %}
 </div>

kitsune/flagit/jinja2/flagit/includes/macros.html

@@ -1,5 +1,5 @@
 {% macro date_by_user(date, user) -%}
  {% trans date=datetimeformat(date, format='longdatetime'), user=display_name(user), url=profile_url(user) %}
- {{date}} by <a href="{{url}}">{{user}}</a>
+ {{date}} by <a rel="nofollow" href="{{url}}">{{user}}</a>
  {% endtrans %}
 {%- endmacro %}

kitsune/flagit/jinja2/flagit/queue.html

@@ -40,7 +40,7 @@ <h3 class="sumo-card-heading"><br>{{ _('Update Status:') }}</h3>
  {% endfor %}
 
  <div class="sumo-button-wrap">
- <a class="sumo-button primary-button" href="{{ url('users.deactivation_log') }}">{{ _('View all deactivated users') }}</a>
+ <a class="sumo-button primary-button" rel="nofollow" href="{{ url('users.deactivation_log') }}">{{ _('View all deactivated users') }}</a>
  </div>
  </div>
 {% endblock %}

kitsune/forums/jinja2/forums/forums.html

@@ -35,7 +35,7 @@ <h5 class="sumo-card-heading"><a
  <a href="{{ forum.last_post.get_absolute_url() }}">
  {{ datetimeformat(forum.last_post.created) }}
  </a><br />
- {{ _('by <a class="username" href="{profile_url}">{username}</a>')|fe(profile_url=profile_url(forum.last_post.author), username=display_name(forum.last_post.author)) }}
+ {{ _('by <a class="username" rel="nofollow" href="{profile_url}">{username}</a>')|fe(profile_url=profile_url(forum.last_post.author), username=display_name(forum.last_post.author)) }}
  {% else %}
  {# Not localized because it's not worth localizers time. #}
  No posts.

kitsune/forums/jinja2/forums/includes/forum_macros.html

@@ -19,15 +19,15 @@
  <td class="title"><a
  href="{{ url('forums.posts', forum_slug=_forum.slug, thread_id=thread.id)|urlparams(last=thread.last_post.id) }}">{{ thread.title }}</a>
  </td>
- <td class="author"><a class="username" href="{{ profile_url(thread.creator) }}">{{ display_name(thread.creator) }}</a>
+ <td class="author"><a class="username" rel="nofollow" href="{{ profile_url(thread.creator) }}">{{ display_name(thread.creator) }}</a>
  </td>
  <td class="replies">{{ thread.replies }}</td>
  {% if thread.last_post %}
  <td class="last-post">
  <a href="{{ thread.get_last_post_url() }}">
  {{ datetimeformat(thread.last_post.created) }}
  </a><br />
- {{ _('by <a class="username" href="{profile_url}">{username}</a>')|fe(profile_url=profile_url(thread.last_post.author), username=display_name(thread.last_post.author)) }}
+ {{ _('by <a class="username" rel="nofollow" href="{profile_url}">{username}</a>')|fe(profile_url=profile_url(thread.last_post.author), username=display_name(thread.last_post.author)) }}
  </td>
  {% endif %}
 </tr>

kitsune/forums/jinja2/forums/includes/post.html

@@ -4,13 +4,13 @@
 <section class="avatar-row">
  <div class="avatar-details user-meta">
  <div class="avatar">
- <a href="{{ profile_url(post.author) }}">
+ <a rel="nofollow" href="{{ profile_url(post.author) }}">
  <img src="{{ profile_avatar(post.author) }}" height="45" width="45" alt="{{ display_name(post.author) }}" />
  </a>
  </div>
  <div class="user">
  <div class="asked-by">
- <a class="author-name" href="{{ profile_url(post.author) }}">
+ <a class="author-name" rel="nofollow" href="{{ profile_url(post.author) }}">
  <span class="display-name">{{ display_name(post.author) }}</span>
  {# L10n: {0} is the number of posts. #}
  <span

kitsune/gallery/jinja2/gallery/media.html

@@ -58,7 +58,7 @@ <h1 class="sumo-page-heading">{{ media.title|truncate(length=55, killwords=True)
  </ul>
  <ul id="media-meta" class="topic-article--meta-list">
  <li class="creator">
- {{ _('by <a href="{profile_url}">{user}</a>')|fe
+ {{ _('by <a rel="nofollow" href="{profile_url}">{user}</a>')|fe
  (profile_url=profile_url(media.creator),
  user=display_name(media.creator)) }}
  </li>

kitsune/groups/jinja2/groups/profile.html

@@ -100,12 +100,12 @@ <h2>{{ _('Group Members') }}</h2>
 
 {% macro user_row(user) -%}
  <div class="avatar">
- <a href="{{ profile_url(user) }}">
+ <a rel="nofollow" href="{{ profile_url(user) }}">
  <img src="{{ profile_avatar(user) }}" alt="" />
  </a>
  </div>
  <div class="info">
- <a href="{{ profile_url(user) }}">{{ display_name(user) }}</a>
+ <a rel="nofollow" href="{{ profile_url(user) }}">{{ display_name(user) }}</a>
  {{ private_message(user) }}
  </div>
 {%- endmacro %}

kitsune/kbadge/jinja2/badger/award_detail.html

@@ -23,7 +23,7 @@ <h2>
 
  {% if badge.creator %}
  <dt>{{ _('Creator:') }}</dt>
- <dd><a href="{{ profile_url(badge.creator) }}">{{ badge.creator }}</a></dd>
+ <dd><a rel="nofollow" href="{{ profile_url(badge.creator) }}">{{ badge.creator }}</a></dd>
  {% endif %}
 
  <dt>{{ _('Created:') }}</dt>
@@ -39,7 +39,7 @@ <h2>{{ _('Award details') }}</h2>
  <dl class="badge-details">
  <dt>{{ _('Awarded to:') }}</dt>
  <dd class="awarded_to">
- <a href="{{ profile_url(award.user) }}">
+ <a rel="nofollow" href="{{ profile_url(award.user) }}">
  <img src="{{ profile_avatar(award.user) }}" alt="" title="{{ display_name(award.user) }}">
  <br>{{ award.user }}
  </a>
@@ -53,7 +53,7 @@ <h2>{{ _('Award details') }}</h2>
  {% if award.creator %}
  <dt>{{ _('Awarded by:') }}</dt>
  <dd>
- <a href="{{ profile_url(award.creator) }}">
+ <a rel="nofollow" href="{{ profile_url(award.creator) }}">
  <img src="{{ profile_avatar(award.creator) }}" alt="" title="{{ award.creator }}">
  {{ award.creator }}
  </a>

kitsune/kbadge/jinja2/badger/badge_detail.html

@@ -22,7 +22,7 @@ <h1 class="sumo-page-heading">
 
  {% if badge.creator %}
  <dt>{{ _('Creator:') }}</dt>
- <dd><a href="{{ profile_url(badge.creator) }}">{{ display_name(badge.creator) }}</a></dd>
+ <dd><a rel="nofollow" href="{{ profile_url(badge.creator) }}">{{ display_name(badge.creator) }}</a></dd>
  {% endif %}
 
  <dt>{{ _('Created:') }}</dt>

kitsune/kbforums/jinja2/kbforums/confirm_post_delete.html

@@ -3,8 +3,8 @@
 {# L10n: {t} is the title of the thread. {d} is the name of the document. #}
 {% set title = _('Delete Post | {t} | {d} Discussion | Knowledge Base')|f(t=thread.title, d=document.title) %}
 {% set crumbs = [(document.get_absolute_url(), document.title),
- (url('wiki.discuss.threads', document.slug), _('Discuss')),
-  (url('wiki.discuss.posts', document.slug, thread.id), thread.title),
+ ((url('wiki.discuss.threads', document.slug), 'rel="ugc nofollow"'), _('Discuss')),
+ ((url('wiki.discuss.posts', document.slug, thread.id), 'rel="ugc nofollow"'), thread.title),
  (None, _('Delete Post'))] %}
 
 {% block content %}
@@ -26,7 +26,7 @@ <h1>{{ _('Are you sure you want to delete this post?') }}</h1>
  {{ _('You are about to permanently delete this post. <strong>This cannot be undone!</strong> Are you sure you want to continue?')|safe }}
  </p>
  <input type="submit" value="{{ _('Delete') }}" />
- <a href="{{ url('wiki.discuss.posts', document.slug, thread.id) }}">{{ _('Cancel') }}</a>
+ <a rel="ugc nofollow" href="{{ url('wiki.discuss.posts', document.slug, thread.id) }}">{{ _('Cancel') }}</a>
  </form>
  </div>
 </article>

kitsune/kbforums/jinja2/kbforums/confirm_thread_delete.html

@@ -3,8 +3,8 @@
 {# L10n: {t} is the title of the thread. {d} is the name of the document. #}
 {% set title = _('Delete Thread | {t} | {d} Discussion | Knowledge Base')|f(t=thread.title, d=document.title) %}
 {% set crumbs = [(document.get_absolute_url(), document.title),
- (url('wiki.discuss.threads', document.slug), _('Discuss')),
- (url('wiki.discuss.posts', document.slug, thread.id), thread.title),
+ ((url('wiki.discuss.threads', document.slug), 'rel="ugc nofollow"'), _('Discuss')),
+ ((url('wiki.discuss.posts', document.slug, thread.id), 'rel="ugc nofollow"'), thread.title),
  (None, _('Delete Thread'))] %}
 
 {% block content %}
@@ -28,7 +28,7 @@ <h1>{{ _('Are you sure you want to delete this thread?') }}</h1>
  {{ _('You are about to permanently delete this thread. <strong>This cannot be undone!</strong> Are you sure you want to continue?')|safe }}
  </p>
  <input type="submit" value="{{ _('Delete') }}" />
- <a href="{{ url('wiki.discuss.posts', document.slug, thread.id) }}">{{ _('Cancel') }}</a>
+ <a rel="ugc nofollow" href="{{ url('wiki.discuss.posts', document.slug, thread.id) }}">{{ _('Cancel') }}</a>
  </form>
  </div>
 </article>

kitsune/kbforums/jinja2/kbforums/discussions.html

@@ -32,9 +32,9 @@ <h1 class="sumo-page-heading">{{ _('{l} Knowledge Base Discussions')|f(l=locale_
  <th class="watch">{{ _('Watch') }}</th>
  {% endif %}
  <th class="title">{{ _('Title') }}</th>
- <th class="author{% if sort == 3 %} sort{% endif %}"><a href="{{ request.path|urlparams(sort=3, desc=desc_toggle) }}">{{ _('Author') }}</a></th>
- <th class="replies{% if sort == 4 %} sort{% endif %}"><a href="{{ request.path|urlparams(sort=4, desc=desc_toggle) }}">{{ _('Replies') }}</a></th>
- <th class="last-post{% if sort == 5 %} sort{% endif %}"><a href="{{ request.path|urlparams(sort=5, desc=desc_toggle) }}">{{ _('Last Post') }}</a></th>
+ <th class="author{% if sort == 3 %} sort{% endif %}"><a rel="ugc nofollow" href="{{ request.path|urlparams(sort=3, desc=desc_toggle) }}">{{ _('Author') }}</a></th>
+ <th class="replies{% if sort == 4 %} sort{% endif %}"><a rel="ugc nofollow" href="{{ request.path|urlparams(sort=4, desc=desc_toggle) }}">{{ _('Replies') }}</a></th>
+ <th class="last-post{% if sort == 5 %} sort{% endif %}"><a rel="ugc nofollow" href="{{ request.path|urlparams(sort=5, desc=desc_toggle) }}">{{ _('Last Post') }}</a></th>
  </tr>
  </thead>
  <tbody class="threads">
@@ -79,17 +79,17 @@ <h1 class="sumo-page-heading">{{ _('{l} Knowledge Base Discussions')|f(l=locale_
  thread_title=thread.title,
  document_discussion_url=url('wiki.discuss.threads',thread.document.slug),
  document=thread.document %}
- <a href="{{ thread_url }}">{{ thread_title }}</a><br>
- in <a href="{{ document_discussion_url }}">{{ document }}</a>
+ <a rel="ugc nofollow" href="{{ thread_url }}">{{ thread_title }}</a><br>
+ in <a rel="ugc nofollow" href="{{ document_discussion_url }}">{{ document }}</a>
  {% endtrans %}
  </td>
- <td class="author"><a class="username" href="{{ profile_url(thread.creator) }}">{{ display_name(thread.creator) }}</a></td>
+ <td class="author"><a class="username" rel="nofollow" href="{{ profile_url(thread.creator) }}">{{ display_name(thread.creator) }}</a></td>
  <td class="replies">{{ thread.replies }}</td>
  <td class="last-post">
- <a href="{{ thread.last_post.get_absolute_url() }}">
+ <a rel="ugc nofollow" href="{{ thread.last_post.get_absolute_url() }}">
  {{ datetimeformat(thread.last_post.created) }}
  </a><br/>
- {{ _('by <a class="username" href="{profile_url}">{username}</a>')|fe(profile_url=profile_url(thread.last_post.creator), username=display_name(thread.last_post.creator)) }}
+ {{ _('by <a class="username" rel="nofollow" href="{profile_url}">{username}</a>')|fe(profile_url=profile_url(thread.last_post.creator), username=display_name(thread.last_post.creator)) }}
  </td>
  </tr>
  {% endfor %}

kitsune/kbforums/jinja2/kbforums/edit_post.html

@@ -5,8 +5,8 @@
 {# L10n: {t} is the title of the thread. {d} is the name of the document. #}
 {% set title = _('Edit a post | {t} | {d} Discussion | Knowledge Base')|f(t=thread.title, d=document.title) %}
 {% set crumbs = [(document.get_absolute_url(), document.title),
- (url('wiki.discuss.threads', document.slug), _('Discuss')),
- (url('wiki.discuss.posts', document.slug, thread.id), thread.title),
+ ((url('wiki.discuss.threads', document.slug), 'rel="ugc nofollow"'), _('Discuss')),
+ ((url('wiki.discuss.posts', document.slug, thread.id), 'rel="ugc nofollow"'), thread.title),
  (None, _('Edit a post'))] %}
 {% set scripts = ('forums',) %}
 
@@ -33,7 +33,7 @@ <h1 class="sumo-page-subheading">{{ _('Edit a post') }}</h1>
  data-preview-container-id="post-preview"
  data-preview-content-id="id_content">{{ _('Preview') }}
  </button>
- <a class="sumo-button push-left" href="{{ url('wiki.discuss.posts', document.slug, thread.id) }}">{{ _('Cancel') }}</a>
+ <a class="sumo-button push-left" rel="ugc nofollow" href="{{ url('wiki.discuss.posts', document.slug, thread.id) }}">{{ _('Cancel') }}</a>
  </div>
  </form>
 

kitsune/kbforums/jinja2/kbforums/edit_thread.html

@@ -4,8 +4,8 @@
 {# L10n: {t} is the title of the thread. {d} is the name of the document. #}
 {% set title = _('Edit thread {t} | {d} Discussion | Knowledge Base')|f(t=thread.title, d=document.title) %}
 {% set crumbs = [(document.get_absolute_url(), document.title),
- (url('wiki.discuss.threads', document.slug), _('Discuss')),
- (url('wiki.discuss.posts', document.slug, thread.id), thread.title),
+ ((url('wiki.discuss.threads', document.slug), 'rel="ugc nofollow"'), _('Discuss')),
+ ((url('wiki.discuss.posts', document.slug, thread.id), 'rel="ugc nofollow"'), thread.title),
  (None, _('Edit thread'))] %}
 
 {% block content %}
@@ -24,7 +24,7 @@ <h1>{{ _('Edit thread "{t}"')|f(t=thread.title) }}</h1>
 
  <div class="form-widget submit">
  <button type="submit" class="btn btn-submit big">{{ _('Update thread') }}</button>
- <a class="btn" href="{{ url('wiki.discuss.posts', document.slug, thread.id) }}">{{ _('Cancel') }}</a>
+ <a class="btn" rel="ugc nofollow" href="{{ url('wiki.discuss.posts', document.slug, thread.id) }}">{{ _('Cancel') }}</a>
  </div>
  </form>
  </article>

kitsune/kbforums/jinja2/kbforums/includes/macros.html

@@ -12,7 +12,7 @@
  {% else %}
  {% trans title=document.title, register_url=url('users.fxa_authentication_init') %}
  This forum is a discussion about improving the "{{ title }}" article. If
- you'd like to participate, please <a href="{{ register_url }}">register</a>.
+ you'd like to participate, please <a rel="nofollow" href="{{ register_url }}">register</a>.
  {% endtrans %}
  {% endif %}
  </p>
@@ -22,7 +22,7 @@
  url=url('wiki.discuss.threads', document.parent.slug, locale='en-US') %}
  This forum is a discussion about improving the localization of the
  "{{ title }}" article. If you want to improve the article content,
- go to the <a href="{{ url }}">English article discussion</a>.
+ go to the <a rel="ugc nofollow" href="{{ url }}">English article discussion</a>.
  {% endtrans %}
  </p>
  {% endif %}