diff --git a/Dockerfile b/Dockerfile index f8907d3..676cf09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,3 +31,10 @@ COPY . . RUN pip uninstall -y pyprof RUN pip install --no-cache-dir . + +# Generating the docs requires the docs source so copy that into the L0_docs so +# that it is available when the test runs. +RUN cp VERSION qa/L0_docs/. && \ + cp README.rst qa/L0_docs/. && \ + cp -r docs qa/L0_docs/. + \ No newline at end of file diff --git a/README.rst b/README.rst index 1d8eb35..170d152 100644 --- a/README.rst +++ b/README.rst @@ -70,11 +70,14 @@ branch for this release is `r20.07 Documentation ------------- +A `Quick Start Guide `_ and +`Installation Guide `_ +provide instructions on how to install and use PyProf. + The User Guide can be found in the -`PyProf docs folder `_, and -provides instructions on how to install and profile with PyProf. +`documentation for current release `_. -An `FAQ `_ provides +An `FAQ `_ provides answers for frequently asked questions. Contributing diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..6c99154 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,37 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = PyProf +SOURCEDIR = . +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(O) + +clean: + @rm -fr $(BUILDDIR) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help clean Makefile diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 0000000..5e64697 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,66 @@ + +{% extends "!layout.html" %} + {% block sidebartitle %} {{ super() }} + + + {% endblock %} + + {% block footer %} {{ super() }} + + + {% endblock %} diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..3c1d00c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,231 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('..')) +from builtins import str +import os +import re +import sphinx_rtd_theme +import subprocess +import textwrap + +# -- Project information ----------------------------------------------------- + +project = u'NVIDIA PyProf' +copyright = u'2020, NVIDIA Corporation' +author = u'NVIDIA Corporation' + +version_long = u'0.0.0' +with open("../VERSION") as f: + version_long = f.readline() + +version_short = re.match('^[\d]+\.[\d]+', version_long).group(0) + +git_sha = os.getenv("GIT_SHA") + +if not git_sha: + try: + git_sha = subprocess.check_output(["git", "log", "--pretty=format:'%h'", "-n1"]).decode('ascii').replace("'","").strip() + except: + git_sha = u'0000000' + +git_sha = git_sha[:7] if len(git_sha) > 7 else git_sha + +version = str(version_long + u"-" + git_sha) +# The full version, including alpha/beta/rc tags +release = str(version_long) + +# hack: version is used for html creation, so put the version picker +# link here as well: +version = version + """
+Version select: """ + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.mathjax', + 'sphinx.ext.napoleon', + 'sphinx.ext.ifconfig', + 'sphinx.ext.extlinks', + 'nbsphinx', + 'breathe' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [u'build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# Setup the breathe extension +breathe_projects = { + "BreathePyProf": "./doxyoutput/xml" +} +breathe_default_project = "BreathePyProf" + +# Tell sphinx what the pygments highlight language should be. +highlight_language = 'text' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + 'canonical_url': 'https://docs.nvidia.com/deeplearning/frameworks/pyprof-user-guide/docs/index.html', + 'collapse_navigation': False, + 'display_version': True, + 'logo_only': False, +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'NVIDIAPyProfdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'NVIDIAPyProf.tex', u'NVIDIA PyProf Documentation', + u'NVIDIA Corporation', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'nvidiapyprof', u'NVIDIA PyProf Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'NVIDIAPyProf', u'NVIDIA PyProf Documentation', + author, 'NVIDIAPyProf', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- +extlinks = {'issue': ('https://github.com/NVIDIA/PyProf/issues/%s', + 'issue '), + 'fileref': ('https://github.com/NVIDIA/PyProf/tree/' + + (git_sha if git_sha != u'0000000' else "master") + '/%s', ''),} + +def setup(app): + # If envvar is set then the file is expected to contain a script + # that is added to every documentation page + visitor_script = os.getenv("VISITS_COUNTING_SCRIPT") + if visitor_script: + app.add_js_file(visitor_script) diff --git a/docs/examples.rst b/docs/examples.rst index 0e841e0..ff0cd52 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -20,9 +20,6 @@ Examples This section provides several real examples on how to profile with PyPRrof. - *TODO:* Provide real examples. Everything here should also be added to - a QA L0_ test to lock in the code - Navigate to the lenet example. :: $ cd pyprof/examples diff --git a/docs/faqs.rst b/docs/faqs.rst index c8c3959..4143455 100644 --- a/docs/faqs.rst +++ b/docs/faqs.rst @@ -18,22 +18,20 @@ PyProf FAQs =========== -#. How do I intercept the Adam optimizer in APEX ? :: +**How do I intercept the Adam optimizer in APEX?** :: import pyprof import fused_adam_cuda pyprof.nvtx.wrap(fused_adam_cuda, 'adam') -#. If you are using JIT and/or AMP, the correct initialization sequence is +**What is the correct initialization if you are using JIT and/or AMP?** - #. Let any JIT to finish. - #. Initlialize pyprof ``pyprof.init()``. - #. Initialize AMP. +#. Let any JIT to finish. +#. Initlialize pyprof ``pyprof.init()``. +#. Initialize AMP. -#. How do I profile with ``torch.distributed.launch``? :: +**How do I profile with ``torch.distributed.launch``?** :: - nvprof -f -o net%p.sql \ - --profile-from-start off \ - --profile-child-processes \ + nvprof -f -o net%p.sql --profile-from-start off --profile-child-processes \ python -m torch.distributed.launch net.py \ No newline at end of file diff --git a/docs/install.rst b/docs/install.rst index c4a8d47..eadc432 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -25,12 +25,19 @@ PyProf is available from GitHub. Installing from GitHub ---------------------- - * git clone https://github.com/NVIDIA/PyProf.git +* Clone the git repository :: + + $ git clone https://github.com/NVIDIA/PyProf.git - * navigate to the top level PyProf directory +* Navigate to the top level PyProf directory - * pip install . +* Install PyProf :: - * verify installation is complete via: ``pip list | grep pyprof``. - # Should display ``pyprof x.y`` where ``x.y`` is the PyProf version installed. + $ pip install . + +* Verify installation is complete with pip list :: + + $ pip list | grep pyprof + +* Should display ``pyprof 3.2.0``. \ No newline at end of file diff --git a/docs/profile.rst b/docs/profile.rst index 77deda2..f9e14ab 100644 --- a/docs/profile.rst +++ b/docs/profile.rst @@ -16,9 +16,6 @@ Profiling PyTorch with PyProf ============================= - TODO: this chapter should go into the details of profiling, - including any options. - Overview -------- For FLOP and bandwidth calculations, we use a relatively straightforward approach. @@ -45,11 +42,6 @@ with information that helps root cause performance issues and prioritize optimiz Enable Profiler in PyTorch Network ---------------------------------- - *TODO:* provide more detail about `torch.cuda.profiler`, why it is needed - and how to access it. The follow is cut and pasted from old README and needs - to be expanded. - - Pyprof makes use of the profiler functionality available in `Pytorch `_. The profiler allows you to inspect the cost of different operators @@ -95,7 +87,7 @@ Here's an example: :: Profile with NVprof ------------------- -If you are not using Nvprof, skip ahead to :ref:`section-profile-with-nsys`. +If you are not using Nvprof, skip ahead to `Profile with Nsight Systems`_. Run NVprof to generate a SQL (NVVP) file. This file can be opened with NVVP. @@ -113,7 +105,7 @@ a message such as :: **_ERR_NVGPUCTRPERM The user running does not have permission to access NVIDIA GPU Performance Counters on the target device_** -Please follow the steps described in :ref:`section-profile-hardware-counters`. +Please follow the steps described in `Hardware Counters`_. .. _section-profile-with-nsys: @@ -223,9 +215,9 @@ Profiling GPU workloads may require access to hardware performance counters. Due to a fix in recent NVIDIA drivers addressing CVE‑2018‑6260, the hardware counters are disabled by default, and require elevated privileges to be enabled again. If you're using a recent driver, -you may see the following message when trying to run nvprof: +you may see the following message when trying to run nvprof:: -**_ERR_NVGPUCTRPERM The user running does not have permission to access NVIDIA GPU Performance Counters on the target device._** + _ERR_NVGPUCTRPERM The user running does not have permission to access NVIDIA GPU Performance Counters on the target device._** For details, see `here `_. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 2eaa6ef..a38f617 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,21 +22,12 @@ PyProf is available in the following ways: * As installable python code located in GitHub. -.. _section-quickstart-prerequisites: - -Prerequisites -------------- - - TODO: List any prerequisites, including point to instructions on how to - install either - .. _section-quickstart-installing-from-github: Installing from GitHub ---------------------- -Make sure you complete the steps in :ref:`section-quickstart-prerequisites` -before attempting to install PyProf. See :ref:`section-installing-from-github` +See :ref:`section-installing-from-github` for details on how to install from GitHub .. _section-quickstart-profile-with-pyprof: diff --git a/qa/L0_docs/test.sh b/qa/L0_docs/test.sh new file mode 100644 index 0000000..95f3a62 --- /dev/null +++ b/qa/L0_docs/test.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TEST_LOG="./docs.log" + +rm -f $TEST_LOG +RET=0 + +apt-get update && \ + apt-get install -y --no-install-recommends python3-pip zip doxygen && \ + rm -rf /root/.cache/pip && \ + pip uninstall -y Sphinx && \ + pip3 install --upgrade setuptools wheel && \ + pip3 install --upgrade sphinx==2.4.4 sphinx-rtd-theme==0.4.3 \ + nbsphinx==0.6.0 breathe==4.14.1 + +set +e + +# Set visitor script to be included on every HTML page +export VISITS_COUNTING_SCRIPT=//assets.adobedtm.com/b92787824f2e0e9b68dc2e993f9bd995339fe417/satelliteLib-7ba51e58dc61bcb0e9311aadd02a0108ab24cc6c.js + +(cd docs && rm -f pyprof_docs.zip && \ + make BUILDDIR=/opt/pytorch/pyprof/qa/L0_docs/build clean html) > $TEST_LOG 2>&1 +if [ $? -ne 0 ]; then + RET=1 +fi + +(cd build && zip -r ../pyprof_docs.zip html) +if [ $? -ne 0 ]; then + RET=1 +fi + +set -e + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + cat $TEST_LOG + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit $RET diff --git a/qa/common/check_copyright.py b/qa/common/check_copyright.py index fae4607..019eef6 100755 --- a/qa/common/check_copyright.py +++ b/qa/common/check_copyright.py @@ -24,6 +24,7 @@ 'preprocessed', 'jmx', 'gz', 'caffemodel', 'json') SKIP_PATHS = ('requirements.txt', + 'qa/L0_docs/VERSION', 'LICENSE', 'VERSION')