8 files changed, 224 insertions(+), 14 deletions(-)

A Makefile
A analyze-tests.dot
A build-steps.dot
A build-steps.png
M gdb-bof-cauldron-2019.org
M gdb-bof-cauldron-2019.pdf
M gdb-bof-cauldron-2019.tex
A submit-patch.png

@@ 0,0 1,11 @@
+.PHONY: all clean
+
+PNG_FILES = $(patsubst %.dot,%.png,$(wildcard *.dot))
+
+all: $(PNG_FILES)
+
+%.png: %.dot
+	dot -Tpng $< -o $@
+
+clean:
+	rm -f *.png

@@ 0,0 1,4 @@
+digraph analyzetests {
+	graph [ dpi = 300 ];
+	"Transform .sum file into a Python dictionary
+}

@@ 0,0 1,16 @@
+digraph buildsteps {
+	graph [ dpi = 300; ]
+	"Remove previous build directory" -> "Pull from GDB repository";
+	"Pull from GDB repository" -> "Copy previous .sum (if not try build)";
+	"Copy previous .sum (if not try build)" -> "Configure GDB";
+	"Configure GDB" -> "Compile GDB";
+
+	"Compile GDB" -> "Test GDB (if supported)" [label=" Normal build?"];
+	"Compile GDB" -> "Racy test GDB" [label=" Racy test?"];
+
+	"Test GDB (if supported)" -> "Calculate regressions";
+	"Calculate regressions" -> "Save build results";
+
+	"Racy test GDB" -> "Analyze racy tests";
+	"Analyze racy tests" -> "Save racy test information";
+}

@@ 10,6 10,22 @@
 
 - https://creativecommons.org/licenses/by/4.0/
 
+* Nomenclature
+
+- *Worker*: The node that performs the “build”.  Usually one per
+  physical machine/VM.  For example, =fedora-x86_64-1= or
+  =ubuntu-aarch64=.
+
+- *Factory*: A recipe of how to perform a build.
+
+- *Builder*: An instance of a factory.  For example,
+  =Fedora-x86_64-m64= or
+  =Ubuntu-Aarch64-native-extended-gdbserver-m64=.
+
+- *Scheduler*: Dispatches jobs to a set of builders.  Can be triggered
+  by specific events like a commit in a repository, a /try build/
+  request or like a cronjob.
+
 * How was it?
 
 - GDB Buildbot started in 2015 as a personal project.


@@ 54,6 70,8 @@
 
   - Mark Wielaard: *1* machine (Fedora =s390x=)
 
+  - Kamil Rytarowski: *1* machine (NetBSD =amd64=)
+
 #+BEAMER: \pause
 
 - Test results are stored directly on-disk, and “garbage-collected”


@@ 61,7 79,64 @@
 
 * How does it work?
 
-- 
+[[file:submit-patch.png]]
+
+* How does it work? @@latex:$^2$@@
+
+#+ATTR_LATEX: :width 0.5\textwidth
+[[file:build-steps.png]]
+
+* Racy tests handling (or an attempt to)
+
+- We keep a list of racy tests (detected weekly through the racy
+  build analysis).
+
+#+BEAMER: \pause
+
+- When a racy build finishes, we include the racy tests in the =xfail=
+  file for that builder.
+
+#+BEAMER: \pause
+
+- We then ignore them when doing normal test builds.  However...
+  whac-a-mole.
+
+* Test analysis (a.k.a. finding regressions)
+
+- Transform the current =.sum= file into a Python dict:
+
+  - ~{ 'gdb.base/test1.exp: name1' : 'PASS', 'gdb.base/test1.exp: name2' : 'FAIL', ...}~
+
+#+BEAMER: \pause
+
+- Do the same for the previous =.sum= file.
+
+#+BEAMER: \pause
+
+- Iterate over the current =.sum= file's dictionary and do:
+
+  #+BEAMER: \pause
+
+  - If the current key is =XFAIL='ed (i.e., a racy test), ignore it.
+
+  #+BEAMER: \pause
+
+  - If the current key exists in the new dict:
+
+    - If it has the same value, good (not a regression).
+
+    - If it changed from =PASS= to =FAIL=, bad.  Report as a
+      regression.
+
+    - If it changed from =FAIL= to =PASS=, good.  Update the baseline.
+
+  #+BEAMER: \pause
+
+  - If the current key doesn't exist in the new dict:
+
+    - If it's a =PASS=, good.  Update the baseline.
+
+    - If it's a =FAIL=, bad.  Report as a new failure.
 
 * Notifications
 


@@ 93,8 168,9 @@
 #+BEAMER: \pause
 
 - Better way to store and retrieve test results (current way is
-  “enough” for what we need, but it can certainly be improved).
+  “enough” for what we need, but it can certainly be improved).  See
+  Serhei's work and Keith's work.
 
 #+BEAMER: \pause
 
-- 
+- =make -jN=, racy tests and =gdb.threads=.

@@ 1,4 1,4 @@
-% Created 2019-09-12 Thu 13:03
+% Created 2019-09-14 Sat 15:04
 % Intended LaTeX compiler: pdflatex
 \documentclass[presentation]{beamer}
 \usepackage[utf8]{inputenc}


@@ 31,7 31,7 @@
 
 \maketitle
 
-\begin{frame}[label={sec:orgbad1c50}]{License}
+\begin{frame}[label={sec:org18d2f5e}]{License}
 \begin{itemize}
 \item License: \alert{Creative Commons Attribution 4.0 International License (CC-BY-4.0)}
 


@@ 39,7 39,25 @@
 \end{itemize}
 \end{frame}
 
-\begin{frame}[fragile,label={sec:org4538604}]{How was it?}
+\begin{frame}[fragile,label={sec:org4f17476}]{Nomenclature}
+ \begin{itemize}
+\item \alert{Worker}: The node that performs the “build”.  Usually one per
+physical machine/VM.  For example, \texttt{fedora-x86\_64-1} or
+\texttt{ubuntu-aarch64}.
+
+\item \alert{Factory}: A recipe of how to perform a build.
+
+\item \alert{Builder}: An instance of a factory.  For example,
+\texttt{Fedora-x86\_64-m64} or
+\texttt{Ubuntu-Aarch64-native-extended-gdbserver-m64}.
+
+\item \alert{Scheduler}: Dispatches jobs to a set of builders.  Can be triggered
+by specific events like a commit in a repository, a \emph{try build}
+request or like a cronjob.
+\end{itemize}
+\end{frame}
+
+\begin{frame}[fragile,label={sec:org85a4532}]{How was it?}
  \begin{itemize}
 \item GDB Buildbot started in 2015 as a personal project.
 \end{itemize}


@@ 59,7 77,7 @@ proved too inefficient over time\ldots{}
 \end{itemize}
 \end{frame}
 
-\begin{frame}[fragile,label={sec:org9dadf47}]{And now?}
+\begin{frame}[fragile,label={sec:org8476c6c}]{And now?}
  \begin{itemize}
 \item The master runs in a dedicated VM at \alert{OSCI} (\textbf{O}pen
 \textbf{S}ource \textbf{C}ommunity


@@ 93,6 111,8 @@ Debian Jessie \texttt{s390x})
 \item Edjunior Machado: \alert{1} machine (CentOS 7 \texttt{PPC64LE})
 
 \item Mark Wielaard: \alert{1} machine (Fedora \texttt{s390x})
+
+\item Kamil Rytarowski: \alert{1} machine (NetBSD \texttt{amd64})
 \end{itemize}
 \end{itemize}
 


@@ 104,13 124,95 @@ every week (tests older than 4 months are deleted).
 \end{itemize}
 \end{frame}
 
-\begin{frame}[label={sec:orgd76b986}]{How does it work?}
+\begin{frame}[label={sec:orgc166782}]{How does it work?}
+\begin{center}
+\includegraphics[width=.9\linewidth]{submit-patch.png}
+\end{center}
+\end{frame}
+
+\begin{frame}[label={sec:orgde5b8ca}]{How does it work? $^2$}
+\begin{center}
+\includegraphics[width=0.5\textwidth]{build-steps.png}
+\end{center}
+\end{frame}
+
+\begin{frame}[fragile,label={sec:org7f13e82}]{Racy tests handling (or an attempt to)}
+ \begin{itemize}
+\item We keep a list of racy tests (detected weekly through the racy
+build analysis).
+\end{itemize}
+
+\pause
+
 \begin{itemize}
-\item 
+\item When a racy build finishes, we include the racy tests in the \texttt{xfail}
+file for that builder.
+\end{itemize}
+
+\pause
+
+\begin{itemize}
+\item We then ignore them when doing normal test builds.  However\ldots{}
+whac-a-mole.
 \end{itemize}
 \end{frame}
 
-\begin{frame}[label={sec:orge36c4d4}]{Notifications}
+\begin{frame}[fragile,label={sec:org05ca5bf}]{Test analysis (a.k.a. finding regressions)}
+ \begin{itemize}
+\item Transform the current \texttt{.sum} file into a Python dict:
+
+\begin{itemize}
+\item \texttt{\{ 'gdb.base/test1.exp: name1' : 'PASS', 'gdb.base/test1.exp: name2' : 'FAIL', ...\}}
+\end{itemize}
+\end{itemize}
+
+\pause
+
+\begin{itemize}
+\item Do the same for the previous \texttt{.sum} file.
+\end{itemize}
+
+\pause
+
+\begin{itemize}
+\item Iterate over the current \texttt{.sum} file's dictionary and do:
+
+\pause
+
+\begin{itemize}
+\item If the current key is \texttt{XFAIL}'ed (i.e., a racy test), ignore it.
+\end{itemize}
+
+\pause
+
+\begin{itemize}
+\item If the current key exists in the new dict:
+
+\begin{itemize}
+\item If it has the same value, good (not a regression).
+
+\item If it changed from \texttt{PASS} to \texttt{FAIL}, bad.  Report as a
+regression.
+
+\item If it changed from \texttt{FAIL} to \texttt{PASS}, good.  Update the baseline.
+\end{itemize}
+\end{itemize}
+
+\pause
+
+\begin{itemize}
+\item If the current key doesn't exist in the new dict:
+
+\begin{itemize}
+\item If it's a \texttt{PASS}, good.  Update the baseline.
+
+\item If it's a \texttt{FAIL}, bad.  Report as a new failure.
+\end{itemize}
+\end{itemize}
+\end{itemize}
+\end{frame}
+
+\begin{frame}[label={sec:org0dac85f}]{Notifications}
 \begin{itemize}
 \item To \emph{gdb-testers}: whenever we detect a possible regression in an
 upstream commit.


@@ 136,8 238,8 @@ notifications are not (just look at \emph{gdb-testers}).
 \end{itemize}
 \end{frame}
 
-\begin{frame}[label={sec:orgcf5098b}]{Problems and challenges}
-\begin{itemize}
+\begin{frame}[fragile,label={sec:org903b48e}]{Problems and challenges}
+ \begin{itemize}
 \item Racy testcases.  Perhaps the most difficult/persistent problem?
 \end{itemize}
 


@@ 152,13 254,14 @@ compare test results and find regressions.
 
 \begin{itemize}
 \item Better way to store and retrieve test results (current way is
-“enough” for what we need, but it can certainly be improved).
+“enough” for what we need, but it can certainly be improved).  See
+Serhei's work and Keith's work.
 \end{itemize}
 
 \pause
 
 \begin{itemize}
-\item 
+\item \texttt{make -jN}, racy tests and \texttt{gdb.threads}.
 \end{itemize}
 \end{frame}
 \end{document}