% !!! PLEASE DON'T CHANGE THESE !!! INSTEAD DEFINE YOUR OWN texdirectives.tex !!!
% These flags are for the final camera ready version for POPL
% Draft, full version flags
\newif\ifdraft\drafttrue
\newif\iffull\fullfalse
\newif\ifanon\anonfalse
\newif\iflast\lasttrue
% LINK to the public repository. Change this in texdirectives if necessary!
\newcommand{\pubrepo}[0]{../../../QuickChick/QuickChick}
% !!! PLEASE DON'T CHANGE THESE !!! INSTEAD DEFINE YOUR OWN texdirectives.tex !!!
\makeatletter \@input{texdirectives} \makeatother
\documentclass[acmsmall,screen]{acmart}\settopmatter{}
%% Note: Authors migrating a paper from PACMPL format to traditional
%% SIGPLAN proceedings format should change 'acmlarge' to
%% 'sigplan,10pt'.
%% Some recommended packages.
\usepackage{booktabs} %% For formal tables:
%% http://ctan.org/pkg/booktabs
\usepackage{subcaption} %% For complex figures with subfigures/subcaptions
%% http://ctan.org/pkg/subcaption
\usepackage{xspace}
\usepackage{preamble}
\usepackage{relsize}
\usepackage{mathpartir}
%definitions for the proof section
\include{sem}
\makeatletter\if@ACM@journal\makeatother
%% Journal information (used by PACMPL format)
%% Supplied to authors by publisher for camera-ready submission
\setcopyright{rightsretained}
\acmJournal{PACMPL}
\acmYear{2018} \copyrightyear{2018} \acmVolume{2} \acmNumber{POPL} \acmArticle{45} \acmMonth{1} \acmPrice{}\acmDOI{10.1145/3158133}
%% Copyright information
%% Supplied to authors (based on authors' rights management selection;
%% see authors.acm.org) by publisher for camera-ready submission
%\setcopyright{acmcopyright}
%\setcopyright{acmlicensed}
%\setcopyright{rightsretained}
%\copyrightyear{2017} %% If different from \acmYear
%% Bibliography style
\bibliographystyle{ACM-Reference-Format}
%% Citation style
%% Note: author/year citations are required for papers published as an
%% issue of PACMPL.
\citestyle{acmauthoryear} %% For author/year citations
\def\l{l}
\begin{document}
%% Title information
\title{Generating Good Generators for Inductive Relations}
%% when present, will be used in
%% header instead of Full Title.
%\titlenote{with title note} %% \titlenote is optional;
% %% can be repeated if necessary;
% %% contents suppressed with 'anonymous'
%\subtitle{} %% \subtitle is optional
%\subtitlenote{with subtitle note} %% \subtitlenote is optional;
%% can be repeated if necessary;
%% contents suppressed with 'anonymous'
%% Author information
%% Contents and number of authors suppressed with 'anonymous'.
%% Each author should be introduced by \author, followed by
%% \authornote (optional), \orcid (optional), \affiliation, and
%% \email.
%% An author may have multiple affiliations and/or emails; repeat the
%% appropriate command.
%% Many elements are not rendered, but should be provided for metadata
%% extraction tools.
%% Author with single affiliation.
\author{Leonidas Lampropoulos}
%\authornote{with author1 note} %% \authornote is optional;
%% can be repeated if necessary
%\orcid{nnnn-nnnn-nnnn-nnnn} %% \orcid is optional
\affiliation{
% \position{Position1}
% \department{Department1} %% \department is recommended
\institution{University of Pennsylvania} %% \institution is required
% \streetaddress{Street1 Address1}
% \city{City1}
% \state{State1}
% \postcode{Post-Code1}
\country{USA}
}
\email{llamp@seas.upenn.edu} %% \email is recommended
\author{Zoe Paraskevopoulou}
%\authornote{with author2 note} %% \authornote is optional;
%% can be repeated if necessary
%\orcid{nnnn-nnnn-nnnn-nnnn} %% \orcid is optional
\affiliation{
% \position{Position2a}
% \department{Department2a} %% \department is recommended
\institution{Princeton University} %% \institution is required
% \streetaddress{Street2a Address2a}
% \city{City2a}
% \state{State2a}
% \postcode{Post-Code2a}
\country{USA}
}
\email{zoe.paraskevopoulou@princeton.edu} %% \email is recommended
\author{Benjamin C. Pierce}
%\authornote{with author2 note} %% \authornote is optional;
%% can be repeated if necessary
%\orcid{nnnn-nnnn-nnnn-nnnn} %% \orcid is optional
\affiliation{
% \position{Position2a}
% \department{Department2a} %% \department is recommended
\institution{University of Pennsylvania} %% \institution is required
% \streetaddress{Street2a Address2a}
% \city{City2a}
% \state{State2a}
% \postcode{Post-Code2a}
\country{USA}
}
\email{bcpierce@seas.upenn.edu} %% \email is recommended
%% Paper note
%% The \thanks command may be used to create a "paper note" ---
%% similar to a title note or an author note, but not explicitly
%% associated with a particular element. It will appear immediately
%% above the permission/copyright statement.
%\thanks{with paper note} %% \thanks is optional
%% can be repeated if necesary
%% contents suppressed with 'anonymous'
%% Abstract
%% Note: \begin{abstract}...\end{abstract} environment must come
%% before \maketitle command
\begin{abstract}
Property-based random testing (PBRT) is widely used in the functional
programming and verification communities.
%
For testing simple properties, PBRT tools such as QuickCheck can
automatically generate random inputs of a given type. But for more complex
properties, effective testing often demands generators for random inputs
that belong to a given type {\em and} satisfy some logical condition.
QuickCheck provides a library of combinators for building such generators by
hand, but this can be tedious for simple conditions and error prone
for more complex ones.
%
Fortunately, the process can often be automated.
%
The most prominent method, {\em narrowing}, works by traversing the
structure of the condition, lazily instantiating parts of the data structure
as constraints involving them are met.
% %
% Recent narrowing approaches have seen success in automatically generating
% well-distributed data satisfying functional predicates using interpreted
% solvers.
We show how to use ideas from narrowing to compile a large subclass of Coq's
inductive relations into efficient generators, avoiding the interpretive
overhead of previous implementations.
%
More importantly, the same compilation technique allows us to produce proof
terms certifying that each derived generator is {\em good}---i.e., sound and
complete with respect to the inductive relation it was derived from.
%
We implement our algorithm as an extension of QuickChick, an existing tool
for property-based testing in Coq.
%
We evaluate our method by automatically deriving good generators for the
majority of the specifications in \SF{}, a formalized
textbook on programming language foundations.
\end{abstract}
%% 2012 ACM Computing Classification System (CSS) concepts
%% Generate at 'http://dl.acm.org/ccs/ccs.cfm'.
\begin{CCSXML}
10011007.10011006.10011008
Software and its engineering~General programming languages
500
\end{CCSXML}
\ccsdesc[500]{Software and its engineering~General programming languages}
%% End of generated code
%% Keywords
%% comma separated list
\keywords{Random Testing, Property-based Testing, Coq, QuickCheck,
QuickChick, Narrowing} %% \keywords is optional
%% \maketitle
%% Note: \maketitle command must come after title commands, author
%% commands, abstract environment, Computing Classification System
%% environment and commands, and keywords command.
\maketitle
% \lk{
% \begin{description}
% \item[slant] \the\fontdimen1\font
% \item[interword space] \the\fontdimen2\font
% \item[interword stretch] \the\fontdimen3\font
% \item[interword shrink] \the\fontdimen4\font
% \item[extra space] \the\fontdimen7\font
% \item[xspaceskip] \the\xspaceskip
% \item[hyphenchar] \the\hyphenchar\font
% \end{description}
% }
\section{Introduction}
\label{sec:intro}
Property-based random testing (PBRT) is a popular technique for quickly
discovering software errors.
%
Starting with Haskell's QuickCheck~\cite{ClaessenH00}, property-based
testing tools have spread to a wide variety of languages~\cite{Arts2008,
Lindblad07, Hughes07, PapadakisS11, Pacheco:2007}.
%
The benefits of PBRT are also enjoyed by users of automated theorem provers
like ACL2~\cite{ChamarthiDKM11} and proof assistants like
Isabelle~\cite{Bulwahn12}, Agda~\cite{DybjerHT04}, and, more recently,
Coq~\cite{itp2015}; testing in these settings can save wasting time and
effort on false conjectures~\cite{DybjerHT03}.
For complex properties, setting up PBRT-style testing can involve
substantial work. Particular effort is required for specifications
involving \emph{sparse preconditions}: ones that hold for only a small
fraction of the input space. For example, consider the following property
(written in Coq's internal functional language, Gallina), which states that
inserting an element into a sorted list preserves sortedness:
%
\examplecode{prop_insert}
%
If we test this property by generating random lists, throwing away ones that
are not sorted, and checking the conclusion \lk{sorted (insert x l)} for the
rest---the \emph{generate-and-test} approach---we will waste most of our
time generating and discarding unsorted lists; worse, the distribution of
the lists that we do {\em not} discard will be strongly skewed toward short
ones, which might fail to expose bugs that only show up for larger inputs.
To tackle properties with sparse preconditions, QuickCheck provides a
comprehensive library of combinators for writing \emph{custom generators}
for well-distributed random values. Such generators are heavily---and
successfully---employed by QuickCheck users. However, writing them can be
both complex and time consuming, sometimes to the point of being a research
contribution in its own right~\cite{TestingNI, testing_ni_jfp,PalkaAST11}!
This has led to interest in automated techniques for enumerating or randomly
generating data structures satisfying some desired
condition~\cite{GligoricGJKKM10, Bulwahn12smartgen, KurajK14,
ClaessenFLOPS14, FetscherCPHF15, LuckPOPL}. One particularly successful
technique is {\em narrowing}~\cite{Antoy94aneeded}, a concept borrowed from
functional logic programming. The idea of narrowing is to construct a
random data structure lazily while traversing the definition of the
predicate it must satisfy.
%
For example, consider the \lk{sorted} predicate:
a list with at most one element is always sorted, while a list with at least two
elements \lk{(x::y::ys)} is sorted if \lk{x} is smaller than \lk{y}
and \lk{(y::ys)} is itself sorted.%
\footnote{Strictly speaking,
this definition is not legal in Gallina, since \lk{y::ys} is not
recognized as a strict subterm of \lk{l}. Expert Coq readers will know
how it can be massaged to make the termination checker happy; others can
ignore this detail.}
%
\examplecode{sorted}
%
%
To generate a list \lk{l} satisfying \lk{sorted l} using narrowing, we look
first at the pattern match and choose randomly whether to instantiate \lk{l}
to an empty list, a list of one element, or a list with at least two
elements. In the first case, we have a value satisfying the predicate and
we are done. In the second, we have a free choice for the value of \lk{x}
and then we are done. In the third case, we next encounter the constraint
\lk {x <=? y}; we generate values for \lk{x} and \lk{y} to satisfy this
constraint locally and then proceed recursively to generate a value for
\lk{ys} satisfying \lk{sorted (y::ys)}. The next time we explore
\lk{sorted}, the parameter \lk{l} is partially instantiated---it consists of
the known value \lk{y} consed onto the unknown value \lk{ys}; this means
that we cannot choose the empty branch; if we choose the second branch we do
not have to generate \lk{x}; and if we choose the third branch we only have
to generate the second element of the list (to be bigger than the first) and
proceed recursively.
%
Automatic narrowing-based generators can achieve testing effectiveness
(measured as bugs found per test generated) comparable to hand-written
custom generators, even for challenging examples~\cite{ClaessenFLOPS14,
FetscherCPHF15, LuckPOPL}.
Unfortunately, both hand-written and narrowing-based automatic generators
are subject to bugs. For hand-written ones, this is because generators for
complex conditions can often be complex, often more than the condition
itself; moreover, they must be kept in sync if the condition is changed,
another source of bugs. Automatic generators do not suffer from the latter
problem, but narrowing solvers are themselves rather complex beasts, whose
correctness is therefore questionable. (The tool of \citet{LuckPOPL} does
come with a proof, but only for an abstract model of part of the core
algorithm, not to the full implementation.)
Bugs in generators can come in two forms: they can generate too much, or too
little---i.e., they can be either unsound or incomplete. Unsoundness can
lead to false positives, which can waste significant amounts of time.
Incompleteness can lead to ineffective testing, where certain bugs in the
program under test can never be found because the generator will never
produce an input that provokes them. Both problems can be
detected---unsoundness by double-checking whether generated values satisfy
the property, incompleteness by techniques such as mutation
testing~\cite{JiaH11}---and unsoundness can be mitigated by filtering away
generated values that fail the double-check, but incompleteness bugs can
require substantial effort to understand and repair.
The core contribution of this paper is a method for compiling a large class
of logical conditions, expressed as Coq inductive relations, into random
generators together with soundness and completeness proofs for these
generators. (We do {\em not} prove that the compiler itself is correct in
the sense that it can only produce good generators; rather, we adopt a {\em
translation validation} approach~\cite{DBLP:conf/tacas/PnueliSS98} where
we produce a checkable certificate of correctness along with each
generator.) A side benefit of this approach is that, by compiling inductive
relations into generators, we avoid the interpretive overhead of existing
narrowing-based generators. As discussed by~\citet{LuckPOPL}, this overhead
is one of the reasons existing generators can be an order of magnitude
slower than their hand-written counterparts.
We have implemented our method as an extension of QuickChick, a QuickCheck
variant for PBRT in Coq~\cite{itp2015}. Using QuickChick, a Coq user can
write down desired properties like
%
\begin{inlinecode}
Conjecture preservation : forall (t t' : tm) (T : ty),
|- t \in T -> t ===> t' -> |- t' \in T.
\end{inlinecode}
and look for counterexamples with no additional effort:\iflast\bcp{We should
actually show the needed ``Derive'' commands.} \fi
{\small
\[
\ \ \mathtt{QuickChick\ \ preservation.\ \ }
\longrightarrow
\mbox % \fbox
{%
$
\begin{array}[t]{l}
\mbox{\it\tt QuickChecking\ \ preservation...\ \ \ Passed\ \ 10000\ \ tests}
\end{array}
$
}
\]
}
Our technical contributions are as follows:
\begin{itemize}
\item We present a narrowing-inspired method for compiling a large class of
inductive definitions into random generators.
%
Section~\ref{sec:example} introduces our compilation algorithm through a
sequence of progressively more complex examples; Section~\ref{sec:algorithm}
describes it in full detail.
%
\item We show how this algorithm can also be used to produce proofs of
(possibilistic) correctness for every derived generator
(Section~\ref{sec:proofs}). Indeed, by judicious application of Coq's
typeclass features, we can use exactly the same code to produce both
generators and proof terms.
%
\item We implement the algorithm in QuickChick, further
integrating testing and proving in the Coq proof assistant and providing
more push-button-style automation while retaining customizability
(Section~\ref{sec:impl}).
%
\item To evaluate the applicability of our method, we applied the QuickChick
implementation to a large part of \SF{}~\cite{SF}, a machine-checked textbook on
programming language theory. Of the 232 nontrivial theorems we considered, 84\%
are directly amenable to PBRT (the rest are higher-order properties that would
at least require significant creativity to validate by random testing); of
these, 83\% can be tested using our algorithm. We discuss these findings in
detail in Section~\ref{sec:eval-sf}.
%
\item To evaluate the efficiency of our generators, we compare them to fine-tuned
handwritten generators for information-flow control abstract machines~(Section~\ref{sec:eval-ifc}). The
derived generators were $1.75\times$ slower than the custom ones, demonstrating
a significant speedup over previous interpreted approaches such as
Luck~\cite{LuckPOPL}.
\end{itemize}
%
Section~\ref{sec:quickchick} introduces QuickChick and provides necessary
background and notations for the rest of the paper.
%
Section~\ref{sec:related} discusses related work. We conclude and draw
directions for future work in Section~\ref{sec:concl}.
\section{QuickChick : QuickCheck in Coq}
\label{sec:quickchick}
We begin with an overview of property-based random testing within Coq using
QuickChick~\cite{QuickChickCoq14}, introducing its basic notations and
typeclasses. Our running example for this section and the rest of the
paper will be the following data type of binary \lk{Tree}s of natural
numbers:
%
\examplecode{nat_tree}
%
Consider the following simple function that completely mirrors its input
tree, that is it interchanges the left and right children of all \lk{Node}s:
%
\examplecode{mirror_tree}
A natural property we expect to hold is that \lk{mirror} is an {\em involution}:
mirroring a tree twice yields the original tree. Armed with an equality on trees
% (\lk{eqTree}), \zoe{Write the type of {\tt eqTree} so that it's obvious that it
% has a bool type? Also in the actual definition we use the corresponding
% notation.} No need for eqTree, just (==)
we can easily state this property in Coq:
%
\examplecode{involution_prop}
%
In order to test the property in QuickChick, we require three things: a
generator for random data, a printer for counterexamples, and
a shrinker for finding minimal counterexamples. In this paper we focus on
generators; see~\citet{ClaessenH00} for discussion of the others.
\subsection{Generators}
\label{sec:generators}
Just like QuickCheck, QuickChick provides a variety of generator combinators
that facilitate the generation of complex data structures. For example, we can
encode a simple, naive generator for binary trees that flips a coin to decide
whether to create a \lk{Leaf} or a \lk{Node}, and proceeds recursively if necessary.
%
\examplecode{naive_gen}
%
The \lk{oneOf} combinator serves the role of the coin flip; it takes a list of
generators and picks one of them at random. Here we have two generators: the
first generator always returns a \lk{Leaf} in the generator monad \lk{G},
described below; the
second one produces an \lk{arbitrary} number and the left and right
subtrees \lk{l} and \lk{r} recursively, before combining the generated data
into a \lk{Node}. We use the monadic \lk{do!} notation provided by
QuickChick to sample the different variables in sequence.
Unfortunately, the \lk{genTree} generator as written does not actually work:
there is no guarantee that it terminates; indeed, the expected size of
generated trees is infinite!
%
In the random testing community, to overcome this limitation, it is common to
introduce a \lk{size} parameter to limit the size of generated terms. In the
following second attempt at a generator, we use a natural number \lk{size} to
signify the maximum depth of the generated tree. In the rest of the paper we
will refer to size parametric generators as \emph{bounded} generators.
%
\examplecode{gen_tree_sized}
%
In the above bounded generator, when the size is
zero, the only tree that can be generated is a \lk{Leaf}. When the size is
nonzero, we have a choice: either generate a \lk{Leaf}, or generate a \lk{Node}
where the left and right subtrees \lk{l} and \lk{r} are generated recursively
with \lk{size} decremented by 1. This choice is made using the \lk{freq}
combinator (short for \lk{frequency}); \lk{freq} takes a list of {\em weighted}
generators and picks one of them, based on the induced discrete
distribution. For example, \lk{gen\_tree\_sized} creates a \lk{Leaf}
$\frac{1}{\texttt{size} + 1}$ of the time and a \lk{Node}
$\frac{\texttt{size}}{\texttt{size} + 1}$ of the time. The \lk{freq} combinator
gives the user a degree of local distribution control that can be used to
fine-tune the distribution of generated data, a crucial feature in
practice.
% \leo{I think that we've used ``a crucial feature in practice'' a few
% times now...}\asz{I grepped for ``crucial'' (2017-07-03 18h36) and this is the
% only place that word shows up in the whole paper.}\leo{I meant across papers (luck, urns?, here...)}
Once we have a bounded generator we can obtain an \emph{unbounded} one using
the \lk{sized} combinator. To understand what \lk{sized} does we must first
peek at the definition of the \lk{G} monad.
%
\verifcode{gen}
%
\lk{G} is represented as a ``reader'' monad with two parameters, a
random seed and a size parameter. When QuickChick runs a computation in
the \lk{G} monad to generate random elements, it will use increasingly larger
size parameters until either a counterexample is found or a predefined size
limit is reached. Given a bounded generator, \lk{sized} will apply it to the
size parameter that is internal to the representation of the generator, making
it implicitly bounded.
%
\verifcode{sized}
%
% \subsection*{Shrinker}
%
% Once our generator has produced a random tree, QuickChick proceeds to test
% whether \lk{mirror\_involution} holds. If it does, it tries again and again
% until it reaches a pre-specified number of tests. If it fails, it reports that
% it found a counterexample. However, the generated inputs can potentially be very
% large containing a lot of noise in addition to a fault-inducing small
% subterm. As an example, imagine that we had made a typo in the \lk{Node} branch
% of the \lk{mirror} function, mirroring the left subtree twice:
% %
% \examplecode{faulty_mirror_small}
% %
% \leo{Show the whole thing?}
% If we ask QuickChick to report the first counterexample found, it responds with
% a large, complex tree:
% %
% \examplecode{generated_tree}
% %
% At this point we know that there is some bug with our program, but the
% counterexample doesn't really help identify it. That is where {\em shrinking}
% comes in.
%
% Shrinking attempts to reduce a known counterexample to a minimal one, using a
% best-first search approach. QuickChick can be given a \lk{shrink} function, that
% given some input \lk{x} produces a list of elements of the same type that are
% ``smaller'' than \lk{x}. QuickChick then uses the \lk{shrink} function on the
% initial counterexample and tries all the potentially smaller shrinks. If it
% finds one, it repeats this process until it has reached a local minimum, which
% is then reported.
% %
% For example, after providing a tree shrinking function, QuickChick returns a
% much simpler counterexample for the faulty mirror property:
% %
% \examplecode{test_result}
% %
% \leo{Small explanation of reasoning with smaller bug?}
%
% But what is a good tree shrinking function? First of all, a \lk{Leaf} can't be
% shrunk to anything, so shrinking a \lk{Leaf} results in an empty list. If the
% input tree is a \lk{Node}, we can either return the left or right subtrees
% directly, or choose to shrink the payload or the left or right subtrees and
% recombine the results. This approach is implemented in the
% following \lk{shrink\_tree} function:
% %
% \examplecode{shrink_tree}
\subsection{QuickChick Typeclasses}
\label{sec:class_hierarchy}
\paragraph*{Generation}
The binary trees of this section always contain \lk{nat}s as labels. Most of the
time, however, Coq users would use a polymorphic tree data type instead.
%
Revisiting the bounded \lk{Tree} generator in the polymorphic case, we would
need to generate an arbitrary \lk{x} of the label type. To avoid cluttering the
definitions and calls to \lk{genTreeSized} with specific generators, we leverage
Coq's {\em typeclasses}~\cite{Wadler1989typeclasses,Sozeautypeclasses}.
Specifically, just like Haskell's QuickCheck, QuickChick provides the \lk{Gen}
typeclass with a single method, \lk{arbitrary}, that produces random elements
of the underlying type \lk{A}.
\iflast\leo{Go into the sources and indent?}\bcp{Yes.}\fi
%
\qccode{src/Classes.v}{gen_class}
%
Looking back at \lk{genTreeSized}, the \lk{arbitrary} method we used to
generate \lk{x} comes from the default \lk{Gen} instance for natural numbers.
In addition, unlike Haskell's QuickCheck, QuickChick introduces
another class, \lk{GenSized}: the class of bounded
generators that explicitly depend on a \lk{size} parameter.
%
\qccode{src/Classes.v}{gen_sized_class}
%
QuickChick automatically converts \lk{GenSized} instances to obtain
corresponding \lk{Gen} instances by applying \lk{sized} to the bounded generator
of the \lk{GenSized} class. This is done by adding the following instance
%
\begin{inlinecode}
Instance GenOfGenSized {A} {H : GenSized A} : Gen A :=
{| arbitrary := sized arbitrarySized |}.
\end{inlinecode}
The above instance provides an \lk{arbitrary} method by applying \lk{sized} to
the \lk{arbitrarySized} method of the \lk{GenSized} instance. We will leverage
this relation between \lk{GenSized} and \lk{Gen} during proof generation
(Section~\ref{sec:proofs}).
Just like in Haskell's QuickCheck ecosystem, it is straightforward to automatically derive
\lk{GenSized} instances (as well as instances of similar typeclasses for shrinking
and printing)~\cite{generic-random}. Using the techniques that will be described
in Section~\ref{sec:proofs}, we can in addition provide proofs of correctness of the
derived generators.
\paragraph*{Decidability}
In the context of a proof assistant like Coq, we are faced with a challenge that
is non-existent in a functional setting: non-executable
specifications. Consider, for example, a different formulation of
the \lk{mirror\_involution} property that uses syntactic equality instead.
%
\examplecode{involution_prop2}
%
Since the conclusion of this property is in \lk{Prop}, we cannot actually
execute it for an arbitrary generated tree \lk{t} to decide whether it holds or
not. To that end, QuickChick provides a \lk{Dec} typeclass
over \lk{Prop}s \lk{P}, with a single method \lk{dec} that provides a
proof of either \lk{P} or its negation.
%
\iflast\leo{How do we do the lower tilde in code?}\fi
\begin{inlinecode}
Class Dec (P : Prop) : Type := { dec : {P} + {~ P} }.
\end{inlinecode}
%
A predicate that is an instance of this class can be used as the conclusion of
any property, which is automatically testable via use of typeclass resolution.
In addition, a user can write \lk{P?} to explicitly convert a property to a
boolean value.
\paragraph*{Constrained Generation}
Just like for simple inductive types \lk{arbitrarySized} and \lk{arbitrary} are the methods of
the \lk{GenSized} and \lk{Gen} typeclasses, for constrained generation \lk{arbSizedST} and \lk{arbST} are the methods of the lk{GenSizedSuchThat} and \lk{GenSuchThat} classes.
%
\begin{inlinecode}
Class GenSizedSuchThat (A : Type) (P : A -> Prop) :=
{ arbitrarySizeST : nat -> G (option A) }.
Class GenSuchThat (A : Type) (P : A -> Prop) :=
{ arbitraryST : G (option A) }.
\end{inlinecode}
%
Given an inductive predicate \lk{P} over some type \lk{A}, these typeclasses
provide access to bounded and unbounded generators for elements of \lk{A} that
claim to satisfy \lk{P} directly. However, since in general there may be no such
such elements (we will see an example in the next section) the result type of
these methods is an \lk{option}. Again, once we have an instance
of \lk{GenSizedSuchThat} we can obtain automatically an instance of
\lk{GenSuchThat} by adding the appropriate instance.
To chain different monadic option actions we could use a bind like what we would
get if we were using monad transformers for the \lk{G} and option monads:
%
\examplecode{bindGenOpt1}
%
However, we can specialize the bind to achieve more control over local
backtracking, which can lead to great efficiency gains in
practice~\cite{ClaessenFLOPS14}. Instead of failing when the generator fails, we
can try again, until we reach a pre-specified number of times; here, the
implicit size parameter of the \lk{G} monad.
%
\examplecode{bindGenOpt2}
%
We use \lk{doM!} notation instead of the more common \lk{do!} notation to chain
actions in this way.
%\leo{Need a place to put this. Thoughts?}\bcp{Not sure. Not here, though.}
%The implementation of the algorithm presented in this paper derives instances of
%the \lk{GenSizedSuchThat} typeclass; the algorithm itself supports generation
%with arbitrary modes.
\section{Good Generators, by Example}
\label{sec:example}
While automating generation for simple types is a much-needed addition to
QuickChick, it still doesn't suffice for testing a common type of interesting
properties: properties with preconditions. The main
focus of this paper is to derive {\em correct} generators for simply-typed data
satisfying dependently-typed, inductive invariants.
%
This section uses examples to showcase different behaviors that our
generation algorithm needs to exhibit; the algorithm itself will be described
more formally in the following section. In particular, we are going to give a
few progressively more complex inductive characterizations of trees, and detail
how we can automatically produce a generator for trees satisfying those
characterizations.
\paragraph*{Nonempty trees}
Our first example is nonempty trees, \IE trees that are not just leaves.
%
\examplecode{nonempty}
%
From a user's perspective, we can quickly come up with a generator for nonempty
trees: we just need to create arbitrary \lk{x}, \lk{l} and \lk{r} and combine
them into a \lk{Node}.
%
\examplecode{gen_nonempty}
%
But how could we automate this process?
We know that we want to generate a tree \lk{t} satisfying \lk{nonempty}; that
means that we need to pick some constructor of \lk{nonempty} to
satisfy. Since there is only one constructor, we only have one option,
\lk{NonEmpty}. By looking at the conclusion of the \lk{NonEmpty} constructor we
know that \lk{t} must be a \lk{Node}. This can be described by a
\emph{unification} procedure. Specifically, we introduce an {\em
unknown variable} \lk{t} (similar to logical variables in logic
programming, or unification variables in type inference) plus one unknown
variable for each universally quantified variable of the constructor (here
\lk{x}, \lk{l} and \lk{r}). We then proceed to unify \lk{t} with \lk{(Node x
l r)}. Since there are no
more constraints---we call them ``\sideconditions''---in \lk{NonEmpty}, and since \lk{x},
\lk{l} and \lk{r} are still completely unknown, we instantiate them
arbitrarily (using the \lk{Gen} instance for natural numbers that is
provided by default, as well as the instance for \lk{Tree}s derived
automatically in the previous section).
\paragraph*{Complete Trees}
For our second example of a condition, consider complete trees (also known as
perfect trees): binary trees whose leaves are all at the same
depth. The shape of a complete
tree can be fully characterized by its depth: a complete tree of depth zero is
necessarily a \lk{Leaf}, while a complete tree of depth \lk{n+1} is formed by
combining two complete trees of depth \lk{n} into a \lk{Node}. This is reflected
in the following inductive definition:
%
\examplecode{complete}
%\leo{Note sure if to show the generator code before or after...}\bcp{I
% originally thought after was OK, but now I think before is actually
% better...}\leo{Agreed}
\newcommand{\inm}{\lk{in1}}
Since \lk{complete} has two parameters, we need to decide whether the derived
generator produces all of them or treats some of them as inputs, i.e., we
need to assign {\em modes} to the parameters, in the sense of functional
logic programming. Let's
assume that that first parameter is an input to the generator (called \lk{\inm}), and
we want to generate trees \lk{t} that satisfy \lk{complete \inm{}
t}. Once again,
we introduce an unknown variable \lk{t} (that we want to generate), as well as
an unknown variable for \lk{\inm}: since the generator will receives \lk{\inm}
as an argument, we don't know its actual value at derivation time!
We now have two constructors to choose from to try to satisfy, \lk{CompleteLeaf}
and \lk{CompleteNode}.
%
If we pick \lk{CompleteLeaf} we need to unify \lk{t} with \lk{Leaf}
and \lk{\inm} with \lk{O}. Since \lk{t} is unconstrained at this point, we can
always unify it with \lk{Leaf}. By contrast since we don't know the value
of \lk{\inm} at derivation time, we need to produce a {\em pattern match} on
it: if \lk{\inm} is \lk{O}, then we can proceed to return a \lk{Leaf},
otherwise we can't satisfy \lk{CompleteLeaf}.
On the other hand, if we pick \lk{CompleteNode}, we introduce new
unknowns \lk{n}, \lk{x}, \lk{l} and \lk{r} for the universally quantified
variables. We proceed to unify \lk{t} with \lk{Node x l r} and \lk{m} with \lk{S
n}. Like before, we need to pattern match on \lk{\inm} to decide at runtime if it
is nonzero; we bind \lk{n} in the pattern match and treat it as an input from
that point onward. We then handle the recursive constraints on \lk{l}
and \lk{r}, instantiating both the left and right subtrees with a recursive call
to the generator we're currently deriving. Finally, \lk{x} remains unconstrained
so we instantiate it arbitrarily, like in the \lk{nonempty} tree case.
%
\examplecode{gen_complete}
The \lk{complete} inductive predicate is particularly well-behaved.
%
First of all, for every possible input depth \lk{m} there exists some tree \lk{t}
that satisfies \lk{complete m t}. That will not necessarily hold in the
general case. Consider for example an inductive definition that consists of only
the \lk{CompleteLeaf} constructor:
%
\examplecode{halfcomplete}
%
Once again, we will need to pattern match on \lk{m}, and, if \lk{m} is zero, we
can proceed as in the previous definition of \lk{complete} to return
a \lk{Leaf}. However, if \lk{m} is nonzero there is nothing we can possibly do
to return a valid tree that satisfies \lk{half\_complete}. This is the reason why
our generators return \lk{option}s of the underlying type:
%
\examplecode{gen_halfcomplete}
Secondly, the usage of the input parameter serves as a structurally decreasing
parameter for our fixpoint. In the general case that is not necessarily true and
we will need to introduce a \lk{size} parameter (like we did in the previous
section for simple inductive types), as we will see in the next example.
\paragraph*{Binary Search Trees}
For a more complex example, consider binary search trees: for every node, each
label in its left subtree is smaller than the node label while each label in the
right subtree is larger. In Coq code, we could characterize binary search trees
whose elements are between two extremal values
\lk{lo} and \lk{hi} with the following code:
%
\examplecode{bst}
%
A \lk{Leaf} is always such a search tree since it contains no elements;
a \lk{Node} is such a search tree if its label \lk{x} is between \lk{lo}
and \lk{hi} and its left and right subtrees are appropriate search trees as
well.
\newcommand{\inlo}{\lk{in1}}
\newcommand{\inhi}{\lk{in2}}
The derived generator (tweaked a bit for readability) is as follows; we explain it below:
%
\examplecode{gen_bst}
%
This generator is bounded: just like in the previous section, we use a natural
number \lk{size} to serve as a limit in the depth of the derivation
tree. When \lk{size} is 0 we are only allowed to use constructors that do not
contain recursive calls to the inductive type we're generating. In the binary
search tree example, that means that we can only choose the \lk{BstLeaf}
constructor. In that case, we introduce unknowns \inlo{} and \inhi{} that
correspond to the inputs to the generation, \lk{t} that corresponds to the
generated tree, as well as two unknowns \lk{lo} and \lk{hi} corresponding to the
universally quantified variables of the \lk{BstLeaf} case. We then try to
unify \inlo{} with \lk{lo}, \inhi{} with \lk{hi}, and \lk{t} with \lk{Leaf}.
Since \lk{lo}, \lk{hi} and \lk{t} are unconstrained, the unification succeeds
and our derived generator returns \lk{Some Leaf}.
When \lk{size} is not zero, we have a choice. We can once again choose to
satisfy the \lk{BstLeaf} constructor, which results in the generator returning
\lk{Some Leaf}. We can also choose to try to satisfy the recursive
\lk{BstNode} constructor. After introducing unknowns and performing the
necessary unifications, we know that the end product of this sub-generator will
be \lk{Some (Node x l r)}. We then proceed to process the constraints that are
enforced by the constructor.
To begin with, we encounter \lk{lo < x}. Since \lk{lo} is mapped to the
input \inlo, we need to generate \lk{x} such that \lk{x} is (strictly) greater
than \inlo. We do that by invoking the typeclass method \lk{arbitraryST}for
generating arbitrary natural numbers satisfying the corresponding
predicate. Now, when we encounter
the \lk{x < hi} constraint both \lk{x} and \lk{hi} are instantiated so we need
to {\em check} whether or not the constraint holds. The notation \lk{p?} looks
for a \lk{Dec} instance of \lk{p} to serve as the boolean condition for
the \lk{if} statement. If it does, we proceed to satisfy the rest of the
constraints by recursively calling our generator. If not, we can no longer
produce a valid binary search tree so we must fail, returning \lk{None}.
%% <<<<<<< HEAD
%% There are a few additional new details in the generator. Firstly, instead of
%% using \lk{frequency} to choose between different constructor options, we use a
%% different combinator, \lk{backtrack}. The \lk{backtrack} combinator operates
%% exactly like \lk{frequency} to make the first choice---choosing a generator with
%% type \lk{G (option A)} based on the induced discrete distribution. However,
%% should the chosen generator fail, it backtracks and chooses a different
%% generator until it either exhausts all options or the backtracking limit. This
%% ability to locally revert failing choices fast approaches can lead to great
%% efficiency gains in practice~\cite{ClaessenFLOPS14}.
%% Moreover, we started using \lk{doM!} notation instead of the more
%% common \lk{do!} monadic notation. Since Coq does not provide built-in monad
%% support, it also doesn't provide support for monad
%% transformers.\bcp{Malecha's library does!} The \lk{doM!}
%% notation introduces an additional monadic option layer on top of the generator
%% monad and can be used to provide more control over local backtracking.
%% \bcp{Do we really need it in this section?}
%% \paragraph*{Good Trees}
%% \leo{TODO: title?\bcp{Non-linear constraints? Not sure an outline is needed
%% at all for this section.} Also, should it be mentioned explicitly in the outline?}
%% =======
One additional detail in the generator is the use of the \lk{backtrack}
combinator instead of \lk{frequency} to choose between different constructor
options. The \lk{backtrack} combinator operates exactly like \lk{frequency} to
make the first choice---choosing a generator with type \lk{G (option A)} based
on the induced discrete distribution. However, should the chosen generator fail,
it backtracks and chooses a different generator until it either exhausts all
options or the backtracking limit.\iflast\leo{Tie back to the doM notation?}\fi
\paragraph*{Nonlinearity}
\iflast\asz{Maybe a name other than ``Good'' would be nice\ldots}\fi
As a last example, we will use an artificial characterization of ``good'' trees
to showcase one last difficulty that arises in the context of dependent
inductive types: {\em non-linear patterns}.
%
\examplecode{non_linear}
%
In this example, \lk{goodTree in1 in2 t} only holds if the tree \lk{t} is
a \lk{Leaf} and \lk{in1} and \lk{in2} are equal, as shown by the non-linear
occurrence of \lk{n} in the conclusion of \lk{GoodLeaf}. If we assume that
both \lk{in1} and \lk{in2} will be inputs to our generator, then this will translate
to an equality check in the actual generator.
%
\examplecode{gen_non_linear}
We can see the equality check \lk{in1 = in2 ?} in the derived generator above. We
can also see that the structure of the generator is similar to the one for
binary search trees, even though it seems unnecessary. In particular, we
encounter calls to \lk{backtrack} with a single element list (which is
equivalent to just the inner generator), as well as an unnecessary match on
the \lk{size} parameter with duplicated branch code. This uniform treatment of
generators facilitates the proof term generation of Section~\ref{sec:proofs}. In
addition, we could obtain the simpler and slightly more efficient generators by a
straight-forward optimization pass.
% \leo{Focus on Horn clauses here?}
%
% Our first example is an arbitrary characterization of all \lk{Tree}s as
% ``good''.
%
% \examplecode{good_tree}
%
% To generate such good trees, all we need is to call \lk{arbitrary}.
%
% \examplecode{gen_good_tree}
%
% Our next example demonstrates the need for unification at the derivation level.
% Consider for example the case where only \lk{Leaf}s are good trees.
%
% \examplecode{good_unif}
%
% Then, to generate good \lk{Tree}s we only have one option: to generate a \lk{Leaf}.
%
% \examplecode{gen_good_unif}
%
% Of course, we will need to combine these two approaches.
%
% \examplecode{good_combo}
% \examplecode{gen_good_combo}
%
% Some times, the resulting dependent type of an inductive constructor will impose
% constraints on the non-generated inputs. For example, we might consider that all
% \lk{Leaf}s are good, but only at index 0.
%
% \examplecode{good_match}
%
% In this case, we will need to pattern match on the input {\em at generation
% time} and ensure the input matches the expected value. If it doesn't, there is
% no way to produce a good tree. Hence, our generators return an \lk{option} from
% now on.
%
% \examplecode{gen_good_match}
%
% So far we only considered non-recursive constructors.
%
% \examplecode{good_rec}
%
% To handle potentially recursive constructors we fall back to the pattern
% encountered in the previous section: we define a local fixpoint based on
% a \lk{size} parameter and split base cases from recursive ones.
%
% \examplecode{gen_good_rec}
%
%One difference is that we use the \lk{backtrack} combinator provided by
%QuickChick instead of frequency. While their type signatures are similar
%%
%\begin{inlinecode}
%backtrack : forall {A : Type}, list (nat * G (option A)) -> G (option A),
%\end{inlinecode}
%%
%the main difference is that \lk{backtrack} takes generators that produce {\em
%optional} values. If such a generator fails (that is, it produces a \lk{None}),
%then \lk{backtrack} picks a different generator from the remaining ones.
%This allows a randomized DFS traversal of the state space.
%
%\leo{For the final two generators we might only focus on the interesting generator part
%and only show that part of the code?}
%
%A different approach is required when recursive calls have their arguments
%fixed.
%
%\examplecode{good_prec}
%
%In such a case, we require that there is a decidability procedure for our
%inductive type. \leo{TODO: Explain before hand decidability}
%
%\examplecode{gen_good_prec}
%
%A more interesting case combines recursive generation and checking, and is very
%reminiscent of narrowing (hence our characterization as a generalization of
%narrowing).
%
%\examplecode{good_narrow}
%
%Here, in the dependent generator we first {\em generate} \lk{tree} to ensure it is
%good at index \lk{0}, and then we {\em check} that it is good at index 1.
%
%\examplecode{gen_good_narrow}
\section{Generating Good Generators}
\label{sec:algorithm}
We now describe the generalized narrowing algorithm
more formally.
%\newcounter{subsect}
%\newcommand{\SUBSECT}[1]{\refstepcounter{subsect}
%\paragraph{\ref{sec:algorithm}.\thesubsect: #1}}
\subsection{Input}
Our generation procedure targets simply-typed inductive data that satisfy
dependently-typed inductive relations of a particular form.
%
More precisely, we take as input an inductively defined relation $R$ with $p$
arguments of types $A_1, A_2, \cdots, A_p$, where each $A_i$ is a simple
inductive type.
%
Each constructor $C$ in the definition of $R$ takes as arguments some number
of universally quantified
variables ($\overline{x}$) and some preconditions---each consisting of an
inductive predicate
$S$ applied to constructor expressions (only consisting of constructors and
variables) $\overline{e}$; its conclusion is $R$ itself applied to
constructor expressions $e_{1}, e_{2}, \cdots, e_{p}$.
%
\newcommand{\ourformat}{%
\[
\begin{array}{l}
\mathtt{Inductive} ~ R : A_1 \rightarrow A_2 \rightarrow \cdots
\rightarrow A_p \rightarrow \mathtt{Prop} \quad := \\
\ \ \ \ldots
\quad | \quad C : \forall ~ \overline{x}, ~ \overline{S ~ \overline{e}}
\rightarrow R ~ e_{1} ~ e_{2} ~ \cdots ~ e_{p}
\quad | \quad \ldots
\end{array}
\]
}
\ourformat
%
We demonstrate the applicability of this class in practical situations in
Section~\ref{sec:eval} and discuss possible extensions to this format as future
work (Section~\ref{sec:concl}).
% \bcp{I think this belongs with the Overview. (And, in general, I wonder
% whether we're presenting things in quite the right order yet.)}
% We proceed to handle each constructor of $R$ separately. For a given constructor
% $C$, we first perform a unification which extracts information about the
% structure of the variables that we want to generate for, as well as a list of
% pattern matches we need to perform and equalities that we need to check. We then
% process all side conditions of $C$, that can all impose additional
% constraints. Finally, we combine the derived sub-generators for each constructor
% using \lk{backtrack}. The rest of this section develops the algorithm in detail.
\subsection{Unknowns and Ranges}\label{sec:unknowns-and-ranges}
We first need to formalize {\em unknowns}, which are used to keep track of sets of
potential values that variables can take during generation, similar to logic
variables in functional logic programming. One important difference is that
sometimes unknowns will be provided as {\em inputs} to the generation algorithm;
this means that they can only take a single fixed value, but that value is not
known at derivation time. Looking back at the \lk{complete} trees example, we
knew that \inm{} would be an input to \lk{gen\_complete}. However, when deriving
the generator we could not make any assumptions about \inm{}: we could not freely
unify it with \lk{O} for instance---we had to pattern match against it.
\newcommand*{\undeft}[1]{\ensuremath{\mathit{undef}_{\!#1}}}
\newcommand*{\fixed}{\ensuremath{\mathit{fixed}}}
We represent sets of potential values as {\em ranges}.
%
\[
r := \undeft{\tau} ~ | ~ \fixed ~ | ~ u ~ | ~ C ~ \overline{r}
\]
%
The first option for the range of an unknown is {\em undefined} (parameterized
by a type). The unknowns we want to generate (such as \lk{tree}, in the binary
search tree example) start out with undefined ranges. On the other hand, a range
can also be {\em fixed}, signifying that the corresponding unknown's value
serves as an input at runtime (\inlo{} and \inhi{} in the binary search tree
example). Next, a range of an unknown can also be a different unknown, to
facilitate sharing. Finally, a range can be a constructor $C$ fully applied to a
list of \leo{tuple of?} ranges.
We use a map from unknowns to ranges, written $\kappa$, to track knowledge
about unknowns during generation. For each constructor $C$, we initialize
this map with the unknowns that we want to generate mapped to
$\undeft{\tau}$ appropriate types $\tau$, the rest of the parameters to $R$
mapped to $\fixed$, and the universally quantiified variables of $C$ also
mapped to appropriate undefined ranges.
%
For instance, to generate a \lk{tree} such that \lk{bst\ \inlo\ \inhi\
tree} holds for all \inlo{} and \inhi,
the initial map for the \lk{BstNode} constructor
would contain \inlo{} and \inhi{} mapped to $\fixed$, \lk{tree}
mapped to $\undeft{\texttt{Tree}}$, and the unknowns \lk{lo}, \lk{hi},
\lk{x}, \lk{l}
and \lk{r} introduced by \lk{BstNode} mapped to corresponding undefined
ranges:
%
\newcommand*{\single}[2]{(#1 \mapsto #2)}
\newcommand*{\extend}[3]{\ensuremath{#1 \oplus #2 \mapsto #3}}
{\small
\[
\begin{array}{rcl}
\kappa & := & \single{\lk{\inlo}}{\fixed} \oplus
\single{\lk{\inhi}}{\fixed} \\
& \oplus &
\single{\lk{tree}}{\undeft{\lk{Tree}}} \\
& \oplus & \single{\lk{lo}}{\undeft{\lk{nat}}} \oplus
\single{\lk{hi}}{\undeft{\lk{nat}}} \oplus
\single{\lk{x}}{\undeft{\lk{Nat}}} \oplus
\single{\lk{l}}{\undeft{\lk{Tree}}} \oplus
\single{\lk{r}}{\undeft{\lk{Tree}}}
\end{array}
\]
}
\subsection{Overview}
\label{sec:generic-structure}
We have already hinted at the general structure of the generation algorithm in
Section~\ref{sec:example}.
%First, the user specifies the generation mode they want:
%looking back at the binary search tree example, we required that \inlo{} and \inhi{}
%be inputs to the generation; we could just as well have produced a generator
%that generates both or either of them as well. To specify the mode upon derivation,
%the user invokes a new top-level command:
%\begin{inlinecode}
% Derive ArbitrarySuchThat for (fun tree => bst in1 in2 tree).
%\end{inlinecode}
%Since only \lk{tree} is bound, that signifies that we will try to generate values
%for \lk{tree}, assuming the (implicitly quantified) \inlo{} and \inhi{} are inputs.
Let's assume \lk{in1} $\ldots$ \lk{inn} will be the inputs to the generator and
that \lk{out1} $\ldots$ \lk{outm} will be the outputs. We then produce a bounded
generator that takes \lk{in1} through \lk{inn} as inputs, as well as an
additional natural number parameter \lk{size}:
%
\begin{inlinecode}
Fixpoint aux_arb size in1 ... inn :=
match size with
| O => backtrack [ ... ($w_C$, $g_{C}$) ...]
| S size' => backtrack [ ... ($w_C$, $g_{C}$) ...]
end.
\end{inlinecode}
%
Both when \lk{size} is zero and when it is not, we use \lk{backtrack} to choose
between a number of generators. In the latter case, we have one sub-generator
$g_C$ for each constructor $C$. The former case is nearly the same, except that
the sub-generators that perform recursive calls to \lk{aux\_arb} are filtered
out of the list. The weights to backtrack ($w_c$) can be chosen by the user via
lightweight annotations, similar to the local distribution control of
Luck~\cite{LuckPOPL}, as we will see in the evaluation
section~(\ref{sec:eval-ifc}).
%\iflast\bcp{Give an example of what the annotations look like? Yes.}
%\leo{Not sure how to show them here, so I put a forward pointer}
%\fi
The general structure of each $g_C$ appears in Figure~\ref{fig:gc}.
%
\begin{figure}[b]
{\small
%\raggedright
\begin{flushleft}
for each $(u, C ~ \overline{p})$ in $\mathit{patterns}$:\\
\end{flushleft}
\fbox{\begin{minipage}{.97\textwidth}
\begin{minipage}{0.15\textwidth}
\texttt{match} ~ \texttt{u} ~ \texttt{with} \\
$| ~ C ~ \overline{p} \Rightarrow$ ... \\
\\
\\
\\
\\
\\
\\
\\
\\
\\
$| ~ \_ \Rightarrow $ \texttt{ret ~ None}
\end{minipage}
\begin{minipage}{\textwidth}
for each $(u_1, u_2)$ in $\mathit{equalities}$:\\
\fbox{\begin{minipage}{0.2\textwidth}
\texttt{if ~ }$u_1 = u_2$~\texttt{then} ...\\
\\
\\
\\
\\
\\
\\
\texttt{else ~ ret ~ None}
\end{minipage}
\begin{minipage}{0.6\textwidth}
for each $\mathit{constraint}$ (e.g., \lk{x < hi}):\\
\fbox{\begin{minipage}{0.6\textwidth}
\begin{minipage}{\textwidth}
\texttt{do! x <- arbST ...} (* {\em instantiations} *) \\
\texttt{if (x < hi)? then ... }(* {\em checks} *)\\
\\
\\
\texttt{else ~ ret ~ None}
\end{minipage}
\end{minipage}
\begin{minipage}{0.35\textwidth}
Final result:\\
\fbox{\begin{minipage}{0.8\textwidth}
\texttt{ret ~(Some ~ ... )}
\end{minipage}
}%
\end{minipage}
}%
\end{minipage}
}%
\end{minipage}
\end{minipage}
}
}
\caption{General Structure of each sub-generator}
\label{fig:gc}
\end{figure}
The outer component of every sub-generator will be a sequence of pattern
matches: unification will sometimes signify that we need to match an unknown
against a pattern. For instance, in the case of \lk{complete} trees we needed to
match \inm{} against \lk{O}. Each such pattern match has two branches: one that
is considered successful and allows generation to continue; and one that catches
all other possible cases and fails (returns \lk{None}).
After nesting all possible matches, we need to ensure that any equalities raised
by the unification hold. In the successful branch of the innermost match (if
any), we start a sequence of if-equality-statements. For example, in the case
of \lk{good} trees that were demonstrating non-linear patterns, we checked
that \lk{in1 = in2} before continuing with the generation.
The equalities are followed by a sequence of instantiations and checks that are
enforced by the \sideconditions{} of $C$. Looking back at the binary search tree example,
we needed to generate a random \lk{x} such that \lk{x} was greater than the
lower bound \lk{in1}; we also needed to check whether that generated \lk{x} was
less than the upper bound \lk{in2}.
Finally, we combine all the unknowns that we wanted to generate for in
a \lk{Some} to return them as the final result. Note that, just like in
the \lk{nonEmpty} trees example, we might need to perform a few more
instantiations if some unknowns necessary remain completely unconstrained.
%In the rest of this section we explain each part in detail before formally
%presenting the derivation of $g_C$.
\subsection{Unification}
\label{sec:unif}
\newcommand*{\updatename }{\ensuremath{\mathit{update}}}
\newcommand*{\equalityname}{\ensuremath{\mathit{equality}}}
\newcommand*{\patternname }{\ensuremath{\mathit{pattern}}}
\newcommand*{\freshname }{\ensuremath{\mathit{fresh}}}
The most important component of the derivation algorithm is the unification. For every
constructor $C$ with conclusion $R ~ e_{1} ~ e_{2} ~ \cdots ~ e_{p}$, we convert
each $e_{i}$ to a range and unify it with the corresponding unknown argument of
$R$. For instance, in the binary search tree example, we would unify the
\inlo, \inhi, and \lk{tree} unknowns with \lk{lo}, \lk{hi}, and
\lk{Node x l r} respectively.
\newcommand*{\unifystate}{\ensuremath{\mathit{UnifyState}}}
\newcommand*{\unifymonad}{\ensuremath{\mathit{Unify}}}
\newcommand*{\optionmonad}[1]{\ensuremath{\mathit{option}} ~ #1}
\newcommand*{\Just}[1]{\ensuremath{\mathit{Some}} ~ #1}
\newcommand*{\Nothing}{\ensuremath{\mathit{None}}}
\newcommand*{\Unknown}{\ensuremath{\mathit{Unknown}}}
\newcommand*{\Range}{\ensuremath{\mathit{Range}}}
\newcommand*{\Pattern}{\ensuremath{\mathit{Pattern}}}
\newcommand*{\updatebase}[2]{\ensuremath{\mathit{update} ~ #1 ~#2}}
%\newcommand*{\updatebase}[2]{\ensuremath{#1 \mapsto #2}}
\newcommand*{\updateop}[3]{\ensuremath{#1 [#2 \mapsto #3]}}
\newcommand*{\equalitybase}[2]{\ensuremath{\mathit{equality} ~ #1 ~ #2}}
\newcommand*{\equalityop}[2]{\ensuremath{#1 \bigcup #2}}
\newcommand*{\patternbase}[2]{\ensuremath{\mathit{pattern} ~ #1 ~ #2}}
\newcommand*{\patternop}[2]{\ensuremath{#1 :: #2}}
\newcommand*{\freshbase}{\ensuremath{\freshname}}
\newcommand*{\freshop}[2]{\ensuremath{#1 \bigcup #2}}
\newcommand*{\constraints}{\ensuremath{\mathit{constraints}}}
\newcommand*{\equalities}{\ensuremath{\mathit{equalities}}}
\newcommand*{\patternms}{\ensuremath{\mathit{patterns}}}
\newcommand*{\unknowns}{\ensuremath{\mathit{unknowns}}}
\newcommand*{\updatestate}[3]{\ensuremath{#1 \{ #2 \leftarrow #3 \}}}
\newcommand*{\letin}[2]{\ensuremath{\text{let} ~ #1 = #2 ~ \text{in}}}
\newcommand*{\lookup}[3]{#3 \leftarrow #1[#2]}
\begin{figure}
{\small
\[
\begin{array}{c}
\text{type} ~ \Pattern = \Unknown ~ | ~ C ~ \overline{\Pattern} \\
\\
\text{record} ~ \unifystate =
\left \{
\begin{array}{lcl}
\constraints & : & \mathit{Map} ~ \Unknown ~ \Range\\
\equalities & : & \mathit{Set} ~ (\Unknown, \Unknown)\\
\patternms & : & \mathit{List} ~ (\Unknown, \mathit{Pattern})\\
\unknowns & : & \mathit{Set} ~ \Unknown \\
\end{array}
\right \}
\\
\\
\text{type} ~ \unifymonad ~ a = \unifystate \rightarrow \optionmonad{(a, \unifystate)}
\\
\begin{array}{ll}
\\
\begin{array}{l}
\updatebase{}{} : \Unknown \rightarrow \Range \rightarrow \unifymonad ~ () \\
\updatebase{u}{r} = \lambda s. \\
\quad \letin{\kappa}{\constraints(s)}\\
\quad \Just{( () , ~ \updatestate{s}{\constraints}{\updateop{\kappa}{u}{r}})}\\
\end{array}
&
\begin{array}{l}
\equalitybase{}{} : \Unknown \rightarrow \Unknown \rightarrow \unifymonad ~ () \\
\equalitybase{u_1}{u_2} = \lambda s. \\
\quad \letin{\mathit{eqs}}{\equalities(s)}\\
\quad \Just{(() , ~ \updatestate{s}{\equalities}{\equalityop{\{u_1 = u_2\}}{\mathit{eqs}}})}\\
\end{array}
\\
\\
\begin{array}{l}
\patternbase{}{} : \Unknown \rightarrow \Pattern \rightarrow \unifymonad ~ () \\
\patternbase{u}{p} = \lambda s. \\
\quad \letin{\mathit{ps}}{\patternms(s)}\\
\quad \Just{(() , ~ \updatestate{s}{\patternms}{\patternop{(u,p)}{\mathit{ps}}})}\\
\\
\end{array}
&
\begin{array}{l}
\freshbase ~~ : \unifymonad ~ \Unknown \\
\freshbase = \lambda s. \\
\quad \letin{\mathit{us}}{\unknowns(s)}\\
\quad \letin{\mathit{u}}{\mathit{fresh\_unknown} ~ (us)}\\
\quad \Just{(u , ~ \updatestate{s}{\unknowns}{\freshop{u}{\mathit{us}}})}\\
\end{array}
\end{array}
\end{array}
\]
}
\caption{Unification Monad}
\label{fig:unification-monad}
\end{figure}
The entire unification algorithm is written inside a state-option monad,
presented in Figure~\ref{fig:unification-monad}. To keep track of information
about unknowns we use a $\mathit{Map}$ from $\mathit{Unknown}$s to $\mathit{Range}$s; to track
necessary equalities---like in the \lk{good} tree of the previous section---we
keep a $\mathit{Set}$ of pairs of unknowns; to produce the necessary pattern
matches---like in \lk{complete} trees---we gather them in a $\mathit{List}$; finally, to be
able to produce fresh unknowns on demand, we keep all existing unknowns in
a $\mathit{Set}$.
Each of the four components of the state can be modified through specific
monadic actions. The \updatename~ action sets the range of an unknown;
the \equalityname~ action registers a new equality
check; \patternname~ adds a pattern match; and \freshname{} generates and returns
a new unknown. We write $\kappa[u]$ for the action that looks up an unknown,
and we write $;$ ~ for the monadic bind operation and $\bot$ to signify failure (the
constant action $\lambda
s. ~ \mathit{Nothing}$). \iflast\bcp{Why not put all this in Figure 1? (Also the
exact definitions of bind and return...}\leo{bind is especially long for minimal gain. We are running against space soon...}\fi
The main unification procedure, $\mathit{unify}$, is shown in
Figure~\ref{fig:unif}.
%
At the top level, we only need to consider three cases for
unification---unknown-unknown, constructor-constructor, and
unknown-constructor---because the $e_{1}$ through $e_p$ are constructor
expressions containing only constructors and variables, which are translated to
constructor ranges and unknowns respectively.
%
Most cases are unsurprising; the main important difference from regular
unification is the need to handle potentially fixed---but not statically known---inputs.
\newcommand*{\unifyname}{\mathit{unify}}
\newcommand*{\unify}[2]{\ensuremath{\unifyname ~ #1 ~ #2}}
\newcommand*{\unifyR}[4]{\ensuremath{\mathit{unifyR} ~ (#1, #2) ~ (#3, #4)}}
\newcommand*{\unifyRC}[3]{\ensuremath{\mathit{unifyRC} ~ (#1, #2) ~ (#3)}}
\newcommand*{\constr}[2]{\ensuremath{C_{#1} ~ r_{#1 1} ~ \cdots ~ r_{#1 #2}}}
\newcommand*{\unifyC}[2]{\ensuremath{\mathit{unifyC} ~ #1 ~ #2 }}
\newcommand*{\equal}[2]{\ensuremath{\mathit{equality} ~ (#1 = #2)}}
\newcommand*{\match}[2]{\ensuremath{\mathit{match} ~ #1 ~ #2}}
\newcommand*{\matchauxname}{\ensuremath{\mathit{matchAux}}}
\newcommand*{\matchaux}[1]{\ensuremath{\matchauxname ~ #1}}
\newcommand*{\return}{\mathit{return}}
\newcommand*{\mapM}{\mathit{mapM}}
\newcommand*{\bindM}{{\mathsmaller{>>=}}}
\newcommand*{\ift}{\text{if}}
\newcommand*{\otherwiset}{\text{otherwise}}
\newcommand*{\andt}{\text{and}}
\begin{figure}
{\small
\[
\begin{array}{@{}l}
\begin{array}{@{}l c l}
\unify{u_1}{u_2} & = &
\begin{cases}
\return ~ () & \ift ~ u_1 = u_2 \\
\left .
\begin{array}{@{}l}
\lookup{\kappa}{u_1}{r_1}; ~ \lookup{\kappa}{u_2}{r_2}; \\
\unifyR{u_1}{r_1}{u_2}{r_2}
\end{array}
\right. & \otherwiset
\end{cases} \\
\unify{(\constr{1}{n})}{(\constr{2}{m})} & = &
\unifyC{(\constr{1}{n})}{(\constr{2}{m})}\\
\unify{u_1}{(\constr{2}{m})} & = &
\lookup{\kappa}{u_1}{r_1}; ~ \unifyRC{u_1}{r_1}{\constr{2}{m}}\\
\unify{(\constr{1}{n})}{u_2} & = &
\lookup{\kappa}{u_2}{r_2}; ~ \unifyRC{u_2}{r_2}{\constr{1}{n}}\\
& & \\
\unifyR{u_1}{\undeft{\tau}}{u_2}{r} & = & \updatebase{u_1}{u_2}\\
\unifyR{u_1}{r}{u_2}{\undeft{\tau}} & = & \updatebase{u_2}{u_1}\\
\unifyR{u_1}{u_1'}{u_2}{r} & = & \unify{u_1'}{u_2}\\
\unifyR{u_1}{r}{u_2}{u_2'} & = & \unify{u_1}{u_2'}\\
\unifyR{\_\,}{\constr{1}{n}}{\_\,}{\constr{2}{m}} & = &
\unifyC{(\constr{1}{n})}{(\constr{2}{m})}\\
\unifyR{u_1}{\fixed}{u_2}{\fixed} & = &
\equalitybase{u_1}{u_2} ; ~ \updatebase{u_1}{u_2}\\
\unifyR{u_1}{\fixed}{\_\,}{\constr{2}{m}} & = &
\match{u_1}{(\constr{2}{m})} \\
\unifyR{\_\,}{\constr{1}{n}}{u_2}{\fixed} & = &
\match{u_2}{(\constr{1}{n})} \\
& & \\
\unifyC{(\constr{1}{n})}{(\constr{2}{m})} & = &
\begin{cases}
\mathit{fold} ~ \unifyname ~ \overline{(r_{1i}, r_{2i})}
& \ift ~ C_1 = C_2 ~ \andt ~ n = m \\
\bot & \otherwiset
\end{cases} \\
& &\\
\unifyRC{u}{\undeft{\tau}}{\constr{2}{m}} & = & \updatebase{u_1}{(\constr{2}{m})}\\
\unifyRC{u}{u'}{\constr{2}{m}} & = & \lookup{\kappa}{u'}{r}; ~ \unifyRC{u'}{r}{\constr{2}{m}}\\
\unifyRC{u}{\fixed}{\constr{2}{m}} & = & \match{u}{(\constr{2}{m})}\\
\unifyRC{u}{\constr{1}{n}}{\constr{2}{m}} & = & \unifyC{(\constr{1}{n})}{(\constr{2}{m})}\\
\end{array}\\
\\
\begin{array}{@{}lll}
\match{u}{(\constr{}{n})} &=&
\overline{p} \leftarrow \mapM ~ \matchauxname ~ \overline{r}; ~
\patternbase{u}{(C ~ \overline{p})} \\
\\
\matchaux{(C ~ \overline{r})} &=&
\overline{p} \leftarrow \mapM ~ \matchauxname ~ \overline{r} ;
~ \return ~ (C ~ \overline{p})\\
\matchaux{u} &=&
\begin{array}[t]{@{}l}
\lookup{\kappa}{u}{r}; \\
\begin{array}[t]{@{}lrl}
\mathit{case} ~ r ~ \mathit{of} \\
| ~~ \undeft{\tau} & \Rightarrow & \updatebase{u}{\fixed}\\
| ~~ \fixed & \Rightarrow & u' \leftarrow \mathit{fresh}; ~ \equalitybase{u'}{u}; ~ \updatebase{u'}{u}; ~ \return ~ u'\\
| ~~ u' & \Rightarrow & \matchaux{u'}\\
| ~~ C ~ \overline{r} & \Rightarrow & \overline{p} \leftarrow \mapM ~ \matchauxname ~ \overline{r}; ~ \return ~ (C ~ \overline{p})
\end{array}
\end{array}
\end{array}
\end{array}
\]
}
\vspace{-2ex}
\caption{Unification algorithm}
\label{fig:unif}
\end{figure}
\newcommand{\rcase}[1]{\paragraph*{Case #1:}}
%
\rcase{$u_i \mapsto \undeft{}$}
If the range of either of the unknowns, say $u_1$, is undefined, we update $\kappa$
so that $u_1$ points to $u_2$ instead. From that point on, they correspond to
exactly the same set of potential values. Consider the \lk{goodTree} example of
the previous section, where in the initial map for \lk{GoodLeaf} we have
unknowns \lk{in1} and \lk{in2} as inputs to the generator, \lk{tree} as the
unknown being generated, and \lk{n} introduced by \lk{GoodLeaf}:
%
{\small
$$ \kappa := \single{\lk{in1}}{\fixed} \oplus
\single{\lk{in2}}{\fixed} \oplus
\single{\lk{tree}}{\undeft{\lk{Tree}}} \oplus
\single{\lk{n}}{\undeft{\lk{nat}}} $$
}
%
We first unify \lk{in1} with \lk{n}; since $\lk{n} \mapsto \undeft{\lk{nat}}$ in
the initial map, the unification updates that map such that
$\lk{n} \mapsto \lk{in1}$.
\rcase{$u_i \mapsto u_i'$}
% Unknown range
If either unknown maps to another unknown we recursively try to unify
using the new unknown as input. For example, when we try to unify \lk{in2}
with \lk{n} in the updated map for \lk{GoodLeaf}, we recurse and attempt to
unify \lk{in1} with \lk{in2}.
\rcase{$u_1 \mapsto \constr{1}{n}$ and $u_2 \mapsto \constr{2}{m}$}
% Constructors
If both ranges have some constructor at their head, there are two possibilities:
either $C_1 \neq C_2$, in which case the unification fails, or $C_1 = C_2$ and
$n=m$, in which case we recursively unify $r_{1i}$ with $r_{2i}$ for all $i$. We
maintain the invariant that all the ranges that appear as arguments to any
constructor contain only constructors and unknowns, which allows us to call
$\unify$ and reduce the total number of cases.
\medskip
The last two cases, dealing with $\mathit{fixed}$ ranges, are the most
interesting ones.
\rcase{$u_1 \mapsto \fixed$ and $u_2 \mapsto \fixed$}
If both $u_1$ and $u_2$ map to a $\mathit{fixed}$ range in $\kappa$, then we
need to assert that whatever the values of $u_1$ and $u_2$ are, they are
equal. This will translate to an equality check between $u_1$ and $u_2$ in the
derived generator. We record this necessary check
using $\mathit{equality}$ and proceed
assuming that the check succeeds, setting one unknown's range to the other.
%
Continuing with the \lk{goodTree} example, when we attempt to unify \lk{in1}
and \lk{in2}, both have $\fixed$ ranges. This results in the equality check $n_1
= n_2$ that appears in \lk{gen\_good}.
\rcase{$u_i \mapsto \fixed$ and $u_j \mapsto \constr{}{n}$}
The last possible configuration pairs a $\fixed$ range against a constructor
range $\constr{}{n}$. This will result in a pattern match in the derived
generator.
%
We saw such an example in the previous section in the form of \lk{complete'}.
One branch of the match will be against a representation of the range
$\constr{}{n}$ and lead to success, while the other branch will terminate
the generation with failure in all other cases.
%
To match against $\constr{}{n}$, we will need to convert all of the ranges
$\overline{r}$ to patterns $\overline{p}$, while dealing with potentially
non-linear appearances of unknowns inside the constructor range. This is done by
traversing the ranges $\overline{r}$, applying a helper function $\matchauxname$
to each, and logging the result in the state monad using
$\mathit{pattern}$.
If $r$ is itself a constructor $C$, we need to recursively traverse its ranges,
convert them to patterns $\overline{p}$ and combine them into a single pattern
$C ~ \overline{p}$.
%
If $r$ is an unknown $u$, we look up its range inside the current map. If it is
undefined we can use $u$ as the bound variable in the pattern; we update the
binding of $u$ in the map to be $\fixed$, as it will be extracting information
out of the fixed discriminee. On the other hand, if the range is $\fixed$, we
need to create a $\mathit{fresh}$ unknown $u'$, use that as the pattern variable
and then enforce an equality check between $u$ and $u'$. Finally, the unknown
and constructor cases result in appropriate recursions.
%Just like in the \lk{complete} tree example of the
%previous section, this is translated into a pattern match. This pattern match
%has one success branch, corresponding to $C_i ~ \overline{r}$, and a catch all
%branch which will eventually lead to backtracking. \leo{More detail? Extra
%figure?}
\subsection{Handling \sideconditions}
\label{sec:side-conditions}
Another important part of the derivation of a generator
for a single constructor $C$ is handling all of its \sideconditions. Given a
\sidecondition{} of the form $S ~ e_1 ~ e_2 ~ \cdots ~ e_m$, we once again
identify a few different cases.
\newcommand*{\ucase}[1]{}
% \newcommand*{\ucase}[1]{{\it (#1.)}}
% Exactly one undefined subcomponent.
\ucase{Single undefined unknown}
If there is exactly one undefined variable amongst the $e_{i}$, we need to
instantiate it. That translates either to a call to the generic \lk{arbitraryST}
function, or to a recursive call to the currently derived generator. The bst
predicate provides examples of both: after the unification is
complete, the map $\kappa$ will have the following form:
%
{\small
\[
\begin{array}{rcl}
\kappa & := & \single{\inlo}{\fixed} \oplus
\single{\inhi}{\fixed} \oplus
\single{\lk{tree}}{\lk{Node} ~ \lk{x} ~ \lk{l} ~ \lk{r}}\\
& \oplus & \single{\lk{lo}}{\inlo} \oplus
\single{\lk{hi}}{\inhi} \oplus
\single{\lk{x}}{\undeft{\lk{Nat}}} \oplus
\single{\lk{l}}{\undeft{\lk{Tree}}} \oplus
\single{\lk{r}}{\undeft{\lk{Tree}}}
\end{array}
\]
}
\iflast\asz{Maybe give more examples of what happens to this map step-by-step as the
algorithm proceeds?}\fi
%
When processing the \sidecondition{} \lk{lo < x}, the unknown \lk{lo} maps
to \inlo which in turn is $\fixed$, while \lk{x} is still undefined. Thus, to
generate \lk{x} such that \lk{lo < x} holds, we need to invoke
the \lk{arbitraryST} method of \lk{GenSuchThat} for \lk{(fun x => in1 < x)}.
After processing this constraint, the range of \lk{x} becomes to $\fixed$:
we know that it has a concrete value but not what it is. For all intents and
purposes it can be treated as if it was an input to the generation from this
point on.
%
%\iflast\leo{Highlight the change somehow? bf doesnt seem to work. Color?}\bcp{Gray?}\fi
{\small
\[
\begin{array}{rcl}
\kappa & := & \single{\inlo}{\fixed} \oplus
\single{\inhi}{\fixed} \oplus
\single{\lk{tree}}{\lk{Node} ~ \lk{x} ~ \lk{l} ~ \lk{r}}\\
& \oplus & \single{\lk{lo}}{\inlo} \oplus
\single{\lk{hi}}{\inhi} \oplus
\single{\lk{x}}{{\textcolor{gray}{\fixed}}} \oplus
\single{\lk{l}}{\undeft{\lk{Tree}}} \oplus
\single{\lk{r}}{\undeft{\lk{Tree}}}
\end{array}
\]
}
Therefore, when processing the \lk{bst lo x l}, only \lk{l} is
unconstrained. However, since generating \lk{l} such that \lk{bst lo x l} holds
is exactly the generation mode we are currently deriving, we just make a
recursive call to \lk{aux\_arb} to generate \lk{l}.
\ucase{No undefined unknowns}
The second possibility for a \sidecondition{} is that all expressions $e_i$ are
completely fixed, in which case we can only check whether this \sidecondition{}
holds. For example, when we encounter the $\lk{x < hi}$ constraint, both \lk{x}
and \lk{hi} have already been instantiated and therefore we need to check
whether \lk{x < hi} holds at runtime, using the \lk{dec} method of the
decidability typeclass.
\ucase{Two or more undefined unknowns}
A final possibility is that a \sidecondition{} could contain multiple undefined
unknowns.
%
Deciding which of them to instantiate first and how many at a
time is a matter of heuristics.
%
For example, if in the constraint \lk{bst lo hi t}, if all of \lk{lo},
\lk{hi} and \lk{t} were undefined, we could pick to make a call to \lk{arbitraryST
bst}, or we could instantiate arguments one at a time. In our
implementation, we prioritize recursive calls whenever possible; we leave
further exploration and comparison of different heuristics as future work.
\subsection{Assembling the Final Result}
\label{sec:final-assembly}
After processing all \sideconditions{} we have an updated constraint map $\kappa$,
where, compared to the constraint map after the unification, some unknowns have
had their ranges $\fixed$ as a result of instantiation. However, there might
still be remaining unknowns that are undefined. Such was the case for
the \lk{nonEmpty} tree example where \lk{x}, \lk{l} and \lk{r} were all still
undefined. Thus, we must iterate through $\kappa$, instantiating any unknowns
$u$ for which $\kappa[u] = \undeft{}$.
%
To complete the generator $g_C$ for a particular constructor, we look up the
range of all unknowns that are being generated, convert them to a Coq
expression, group them in a tuple and return them.
%
\newcommand*{\logger}[1]{\mathit{log} ~ \mathtt{#1}}
\newcommand*{\emitpatterns}{\mathit{emit\_patterns}}
\newcommand*{\emitequalities}{\mathit{emit\_equalities}}
\newcommand*{\emitconstraints}{\mathit{emit\_\sideConditions}}
\newcommand*{\finalassembly}{\mathit{final\_assembly}}
\newcommand*{\emitresult}{\mathit{emit\_result}}
\newcommand*{\finalcall}{\mathit{emit\_final\_call}}
\newcommand*{\logg}{}
\begin{figure}
{\small
\[
\begin{array}{l}
\text{let} ~ \kappa = \overline{in \mapsto \fixed}
\oplus {out \mapsto \undeft{\tau_{out}}}
\oplus \overline{x \mapsto \undeft{\tau_x}} ~ \text{in} \\
\text{let} ~ st = \left \{
\begin{array}{lcl}
\constraints & = & \kappa \\
\equalities & = & \emptyset\\
\patternms & = & [] \\
\unknowns & = & \overline{in} \bigcup \{out\} \bigcup \overline{x}
\end{array}
\right \} ~ \text{in}
\\
\text{case} ~ \mathit{runUnifyState} ~ \{\unify{e_1}{e_1'}; ~\unify{e_2}{e_2'}; ~\ldots ; ~\unify{e_p}{e_p'}\}~ st ~ \text{of} \\
\quad| ~ \Nothing \rightarrow \logg~ \fbox{\texttt{ret ~ None}}\\
\quad| ~ \Just{(st', ())} \rightarrow \emitpatterns ~ (\patternms ~ st') ~ (\equalities ~ st') ~ (\constraints ~ st')\\
\\
\\
\begin{array}{ll}
\begin{array}{l}
\emitpatterns ~ [] ~ eqs ~ \kappa = \emitequalities ~ eqs ~ \kappa\\
\emitpatterns ~ ((u,p) :: ps) ~ eqs ~ \kappa = \\
\quad \logg~
\quad\fbox{\begin{minipage}{0.3\textwidth}
$\begin{array}{l}
\texttt{match~}u\texttt{~with}\\
\texttt{|~}p\texttt{~=>~} \emitpatterns ~ ps ~ eqs ~ \kappa \\
\end{array}$
\end{minipage}}\\
\end{array} &
\begin{array}{l}
\emitequalities ~ [] ~ \kappa = \emitconstraints ~ (\overline{S ~ \overline{e}}) ~ \kappa \\
\emitequalities ~ ((u_1, u_2) :: eqs) ~ \kappa = \\
\quad \logg~
\quad\fbox{\begin{minipage}{0.38\textwidth}
$\begin{array}{l}
\texttt{if~ (}u_1 \texttt{=} u_2\texttt{)? ~ then} ~ \emitequalities~eqs~\kappa\\
\texttt{else~ret~None}\\
\end{array}$
\end{minipage}}\\
\end{array}
\end{array}\\
\\
\begin{array}{ll}
\begin{array}{l}
\emitconstraints ~ [] ~ \kappa = \finalassembly ~ \kappa\\
\emitconstraints ~ ((S ~ \overline{e})::ss) ~ \kappa = \\
\quad \text{if} ~ \forall ~ u \in \overline{e}. ~ \kappa[u] \neq \undeft{} ~ \text{then}~ \\
\quad\quad\quad\logg~ \fbox{\begin{minipage}{0.35\textwidth}
\texttt{if~}$(S~\overline{e})$\texttt{?~then~} $\emitconstraints ~ ss ~ \kappa$\\
\texttt{else~ret~None}
\end{minipage}}\\
\quad \text{else} \\
\quad \text{let} ~ \{u_1, \ldots, u_k\} =
\{ u \in \overline{e} ~ | ~ \kappa[u] = \undeft{} \} ~ \text{in} \\
\quad\quad\quad \logg~\fbox{\begin{minipage}{0.35\textwidth}
\texttt{do!~}$u_1$\texttt{~ <- arbitrary;}\\
\quad $\ldots$\\
\texttt{do!~}$u_{k-1}$\texttt{~ <- arbitrary;}\\
$\finalcall ~ u_k$ \\
$\emitconstraints ~ ss ~ \kappa[u_i \leftarrow \fixed]_{i=1}^k$
\end{minipage}}\\
\end{array}
&
\begin{array}{l}
\finalcall ~ u_k = \\
\quad \text{if}~ u_k \sim out ~ \text{then}\logg~\fbox{\texttt{doM!}~$u_k$~\texttt{<- aux\_arb~size'}}\\[1ex]
\quad \text{else} ~ \logg~\fbox{\texttt{doM!}~$u_k$~\texttt{<- arbitraryST (}$\lambda u_k. ~ S ~ \overline{e})$\hspace{1.15em}}\\
\\
\finalassembly ~ \kappa = \\
\quad \text{let} ~ \{u_1, \ldots, u_f\} =
\{ u | ~ \kappa[u] = \undeft{} \} ~ \text{in} \\
\quad \text{let} ~ \kappa' = \kappa[u_i \leftarrow \fixed]_{i=1}^k ~ \text{in}\\
\quad\quad\quad \logg~\fbox{\begin{minipage}{0.35\textwidth}
\texttt{do!~}u\texttt{~ <- arbitrary;}\\
\texttt{ret~ (Some~}$ \emitresult ~ \kappa' ~ out ~ \kappa'[out]$\texttt{)}
\end{minipage}}\\
\\
\emitresult~\kappa~u~u' = \emitresult~\kappa~u'~\kappa[u']\\
\emitresult~\kappa~u~\fixed = \fbox{$u$}\\
\emitresult~\kappa~u~(C~r_1~\ldots~r_k) = \\
\quad \fbox{$C ~ ( \emitresult ~ \kappa ~ \_ ~r_1 ) ~ \ldots ~ (\emitresult ~ \kappa ~ \_ ~ r_k)$}
\end{array}
\end{array}
\end{array}
\]
}
\caption{Derivation of one case of a generator $g_C$ (for a single
constructor $C$),
in pseudo-code. Boxes delimit ``quasi-quoted'' Coq generator code to be
emitted. Inside boxes, {\em italic} text indicates ``anti-quoted''
pseudo-code whose result is to be substituted in its place.
\iflast\bcp{Shouldn't we put $g_C = $ or something at the top, plus perhaps a
reminder of where $\overline{in}$, etc., come from?}\fi}
\label{fig:full-formal}
\end{figure}
\subsection{Putting it All Together}
\label{sec:finalassembly}
A formal presentation of the derivation for a single constructor is shown in
Figure~\ref{fig:full-formal}. Here, for simplicity of exposition, we allow only
a single output $out$. In general, even though our implementation of the algorithm
deals with a single output as well, the algorithm presented in this section can
handle an arbitrary number of outputs.
Given an inductive relation $R$ and a particular constructor
$
C : \forall ~ \overline{x}, ~ \overline{S ~ \overline{e}}
\rightarrow P ~ e_{1} ~ e_{2} ~ \cdots ~ e_{p},
$
our goal is to generate $out$ such that for all $\overline{in}$, the predicate
$R ~ e_1' ~ e_2' ~ \ldots ~ e_p'$ holds via constructor $C$, where the $e'$s are
constructor expressions containing only variables in $\{out\} \bigcup \overline{in}$.
%
First, we create an initial map $\kappa$ as described in
Subsection~\ref{sec:unknowns-and-ranges}.
%
We use it to construct an initial state $st$ for the unification monad
(Subsection~\ref{sec:unif}), where the $\patternms$ and $\equalities$ fields are
empty, while the $\unknowns$ field holds $\overline{in}$, $out$ and all
universally quantified variables of $C$.
%
We then evaluate a sequence of monadic actions, each one attempting to unify
$e_i$ with its corresponding $e_i'$. If at any point the unification fails, the
constructor $C$ is not inhabitable and we fail. If it succeeds, we proceed to
produce all of the nested pattern matches and equalities in order
($\emitpatterns$ and $\emitequalities$), as described
in Subsection~\ref{sec:generic-structure}.
%
Afterwards, we process all the \sideconditions{} using $\emitconstraints$ as
described in Subsection~\ref{sec:side-conditions}, emitting instantiations or
checks as appropriate, while updating the constraint set $\kappa$. Finally, we
complete the generation by instantiating all unknowns that are still undefined
and constructing the result by reading off the range of $out$ in the final
constraint set (\ref{sec:finalassembly}).
% \bcp{And can we write down a correctness theorem to clarify what it means
% for this generation procedure to be correct? Or is that exactly what we don't
% have? (Even if it is, we need to discuss this fact.)}\leo{We can write what
% it means for each specific generator to be correct. Which is what the next
% section is all about}.
\section{Generating Correctness Proofs}
\label{sec:proofs}
% \leo{After reading this section, my overall comment is that this section is really good, with one caveat.
% It describes the high-level infrastructure in great detail, and gets most of the
% intuition across really well. However, the reader doesn't get an accurate view
% of the actual correctness proof of the derived bounded generator other than ``it
% is produced by the same algorithm''. Could we have a side-by-side for a small
% predicate (preferrably from section 2) that shows this mirroring? We could
% replace the gory dependent proof terms with dots where they become too
% large. I'm not sure whether that should happen at the beginning (``top-level
% proof'') or at the end of that subsection. I can see merits to both. But something like this
% needs to be in.}\leo{Well.. if the halfcomplete is ~800 lines, I don't see how we could ever do anything of the sort.
% Unless Zoe has any thoughs of a sketch that could work...}
% \bcp{As a relative outsider, I'm afraid I have to disagree that the
% intuition comes across well. There's a lot of {\em low}-level stuff that
% seems reasonably well explained, but the high-level story --- what we are
% trying to accomplish and what are the main steps or ideas involved in
% accomplishing it --- does not come through clearly enough.}
This section describes how we automatically generate proofs that
our generators are sound and complete with respect to the inductive
predicates they were derived from. Following the translation
validation approach of~\citet{DBLP:conf/tacas/PnueliSS98}, rather than
proving once and for all that {\em every} generator we build is guaranteed
to be correct, we build a proof term certifying that each
specific generator is correct at the same time as we build the
generator itself. In fact, the same algorithm that is used to compile
generators from inductive predicates is also used to compile their
corresponding proofs of correctness. We leverage an existing verification
framework for QuickChick, designed to allow users to
(manually) prove soundness and completeness of generators built from
QuickChick's low-level primitives~\cite{itp2015}.
This verification framework assigns semantics to each generator by
mapping it to its set of outcomes, i.e. the elements that have non-zero
probability of being generated. This enables proving that all the elements in
a set of outcomes satisfy some desired predicate (soundness),
and that all the elements that satisfy the predicate are in the set of outcomes
(completeness). To ease reasoning about user-defined generators, QuickChick
provides a library of lemmas that specify the behavior of built-in
combinators.
We leverage this framework to specify the set of outcomes of derived generators:
given an inductive relation and some input parameters, it should be exactly the
set of elements that satisfy the inductive relation. Automatic generation of
proofs is analogous to generation of generators and is done using the same
algorithm. Just as generators are derived by composing generator combinators
that we select by examining the structure of the inductive predicate, proofs are
derived by composing the corresponding correctness lemmas that are provided by
QuickChick. We glue these proof components together in order to obtain proofs
for unsized generators using typeclass resolution, just as we did to obtained
unsized generators. To enable this, we extend the typeclass infrastructure of
QuickChick to encode properties of generators as typeclasses and we
automatically generate instances of these classes for the derived generators.
%% that we want to
%% prove about generators as typeclasses, and we automatically derive instances of
%% these classes for the generators. By providing appropriate instances, we can
%% automatically lift these proofs from bounded to unbounded generators, just as we
%% did with the generators themselves.
%% =======
%% QuickChick gives semantics to a generator as the set of all of its possible
%% random outputs. We use this semantics to specify the set of outcomes of derived
%% generators: the set of elements that satisfy an inductive relation when some of
%% its parameters are given as inputs. QuickChick also provides a collection of
%% lemmas that specify the set of outcomes of built-it combinators. Since our
%% generators are compositionally built up from other generators using QuickChick's
%% combinators, we can use the corresponding lemmas in order to prove their
%% correctness.
%% %As we will explain, obtaining proofs of correctness relies on other
%% %definitions and proofs which we also derive automatically.
%% The generation strategy for proof terms is the same as the one of
%% generators. We extend QuickChick's typeclass infrastructure to encode the
%% properties that we want to prove about generators as typeclasses, and we
%% automatically derive instances of these classes for the generators. By providing
%% appropriate instances, we can automatically lift these proofs from bounded to
%% unbounded generators, just as we did with the generators themselves.
%% >>>>>>> 6ad577513e68054dff04a68c1cc905b4d93a9b79
%% The proof terms that are
%% generated for unbounded generators have the same structure as the generators
%% themselves, and we construct them using the same algorithm.
%% In the rest of this section, we will briefly
%% describe QuickChick's verification framework focusing on the machinery that we
%% use for the proof generation, as well as the proof generation itself.
%% QuickChick also
%% provides a collection of lemmas that specify the set of outcomes of built-it
%% combinators.
%% Applying such correctness lemmas often requires the generators to have certain
%% properties, hence we also have to automatically generate proofs for these
%% properties as well. The proof terms have the same shame as the generators In
%% fact they can be abtain using the same algorithm but replacing combinators and
%% generators with the corresponding proofs.
Subsection \ref{subsec:verif_qc} briefly describes QuickChick's
verification framework, focusing on the proof generation machinery.
\ref{subsec:verif_proofs} outlines the structure of the generated proofs
and describes all the terms, definitions, and proofs that we need to generate
in order to obtain the top-level correctness proof.
\ref{subsec:verif_typeclasses} describes the extensions to the typeclass
infrastructure of QuickChick that we made in order to facilitate proof
generation.
\subsection{Verification Framework}\label{subsec:verif_qc}
QuickChick assigns semantics to generators by mapping them to the set of values
that have non-zero probability of being generated. Recall from
section~\ref{sec:generators} that generators are functions
mapping a random seed and a natural number to an element of the underlying type. The
semantics of a generator for a given size parameter is exactly the values that
can be generated for this particular size parameter.
%
\[ \semGenS{g}{s} = \setcomp{x}{\exists r,~ g~s~r = x}\]
%
We can then define the semantics of a generator by taking the union of these
sets over all possible size parameters.
%
\[ \semGen{g} = \bigcup_{s \in \nat}{\semGenS{g}{s}} \]
%
It may seem as though we could have skipped the first definition and inlined its
right-hand side in the second. However, by separating out the first definition
we can additionally characterize the behavior of generators with respect to the
size parameter. For instance, we can define the class of size-monotonic
generators, whose set of outcomes for a given size parameter is included to the
set of outcomes for every larger size parameter.
%
\[ \sizeMonotonic{g} \eqdef \forall s_1~s_2,~s_1 \leq s_2 \rightarrow \semGenS{g}{s_1} \subseteq \semGenS{g}{s_2} \]
%
Another useful class of generators is bound-monotonic generators, i.e., bounded
generators that behave monotonically with respect to their bound parameter.
Recall that bounded generators are parameterized by a natural number which
bounds the size of the generated terms.
%
\[ \boundMonotonic{g} \eqdef \forall s~s_1~s_2,~s_1 \leq s_2 \rightarrow \semGenS{g~s_1}{s} \subseteq \semGenS{g~s_2}{s} \]
%
Together, these characterizations allow us to obtain convenient
specifications for combinators. To support reasoning about size-monotonicity
properties, QuickChick encodes them as typeclasses and provides lemmas
(encoded as typeclass instances) that various generator combinators are
size monotonic if all the involved generators are size monotonic. For
instance, here is the lemma for monadic binding:
%
\begin{mathpar}
\inferrule*[right=monBind]
{\sizeMonotonic{g} \\
\forall x\in \semGen{g},~\sizeMonotonic{(f~x)}
}
{\sizeMonotonic{(g~\bindM~f)}}
\end{mathpar}
%
There is a similar lemma that guarantees that \lk{sized} is size monotonic but
it requires both bound and size monotonicity for the bounded
generator.
%
\begin{mathpar}
\inferrule*[right=monSized]
{\sizeMonotonic{g} \\
\forall s,~\boundMonotonic{(g~s)}
}
{\sizeMonotonic{(\lk{sized}~g})}
\end{mathpar}
%
To prove that \lk{sized} is monotonic, we also need a second premise that
requires the bounded generator to be bound monotonic. This is because \lk{sized}
will use the internal size parameter of \lk{G} to instantiate the bound
parameter of the generator.
To support reasoning about generators, QuickChick provides a library of lemmas
that specify the semantics of generator combinators and can be used to
compositionally verify user defined generators. These lemmas can be seen as a
proof theory for the \lk{G} monad; one can apply them in order to build
derivations that computations in this monad are correct.
The simplest example of a correctness lemma is the one of \lk{ret}.
Unsurprisingly, the semantics of the return of the \lk{G} monad is just a
singleton set.
% XXX hack
\vspace{-0.7cm}
%
\begin{mathpar}
\inferrule*[right=semRet]
{\empty}
{\semGen{\lk{ret}~x} = \singleton{x}}
\end{mathpar}
%
The lemma for monadic bind is more interesting. In particular, the expected
specification that composes the set of outcomes of the two generators using an
indexed union is true, but under the additional requirement that the generators
involved are size monotonic.
%
\begin{mathpar}
\inferrule*[right=semBind]
{\sizeMonotonic{g} \\
\forall x\in \semGen{g},~\sizeMonotonic{(f~x)} \\
\semGen{g} = s \\
\forall x\in s,~\semGen{f~x} = h~x
}
{\semGen{g~\bindM~f} = \bigcup_{x \in s} {h~x}}
\end{mathpar}
%
The intuition behind
this requirement~\cite{itp2015} is that the set on the left-hand side of
{\sc semBind} contains elements that are generated when the same size
parameter is threaded to both generators, whereas the right-hand side indexes
over elements that have been generated by $g$ when the size parameter ranges
over all natural numbers. To address this mismatch, we use monotonicity. In
particular, to obtain the right to left inclusion we can pick
a witness for the size parameter that is greater than both of the size
parameters we obtain as witnesses from the hypothesis, such as their sum (or
max) and then use monotonicity to prove the inclusion.
The correctness lemma for {\tt sized} is crucial for proof generation,
as it gives us proofs about unbounded generators. It states that
the semantics of the combinator is the union of the sets of outcomes of the
bounded generator indexed over all natural numbers.
%
\begin{mathpar}
\inferrule*[right=semSized]
{\boundMonotonic{g} \\
(\forall x \in \nat,~\sizeMonotonic{(g~x)}) \\
\forall x \in \nat,~\semGen{g~x} = f~x
}
{\semGen{\lk{sized}~g} = \bigcup_{x \in \nat}{f~x}}
\end{mathpar}
%
The lemma requires that the bounded generator is size monotonic for all bounds
and, in addition, that it is monotonic in the bound parameter itself. These
conditions are required because the set on the left-hand size of the
specification contains elements that are generated from $g$ using same number
for the size and the bound, whereas in the right-hand side the bound and the
size parameter range independently over natural numbers. As in the case of the
monadic bind, we can work around this mismatch using monotonicity.
\subsection{Proof Generation}\label{subsec:verif_proofs}
This section describes the proof terms that we generate for each
derived generator. To describe the structure of the constructed proof terms,
we will use the generator for binary search trees presented in
\ref{sec:example} as a running example. The terms themselves have the same
structure as the generators and are generated using using the same algorithms,
replacing the generator building blocks with their proof counterparts. For brevity, we assume a single output of the generation procedure; we can easily
encode multiple outputs using tuples.
\paragraph*{Top-level proof}
Let $R : A \rightarrow Prop$ be an inductive predicate and $g : \lk{option}~A$ a
derived generator for this predicate. We want to generate proofs that $g$ is
sound and complete with respect to this predicate, i.e., that the set of outcomes
of the generator is exactly the elements that satisfy $P$:
\[ \isSome \cap \semGen{g} = \imset{\lk{Some}}{P} \]
Since our generators can fail (their return type is \lk{option}
$A$), we need to take the image of $P$ under \lk{Some}. We also remove
\lk{None}
from the set of outcomes of $g$ by intersecting it with $\isSome$, i.e., the set
of elements whose outermost constructor is \lk{Some}.
\zoe{Write that None's are important but we only verify functional correctness not efficiency...}
In the case of binary search trees, this amounts to saying that the set of
outcomes of the unbounded generator (which we obtain using \lk{sized}; it is
automatically derived by typeclass resolution) is exactly the set of trees
that satisfy the \lk{bst} predicate for some given inputs.
\[ \forall~in_1~in_2,~\isSome \cap \semGen{\lk{sized}~(\lk{gen\_bst}~in_1~in_2)} = \imset{\lk{Some}}{\lk{bst}~in_1~in_2} \]
As expected, to generate proofs about unbounded generators we have to first
generate proofs about bounded generators. These proofs can be then lifted
using the specification of \lk{sized} that we saw in the previous
section.
\paragraph*{Proofs for Bounded Generators} Before
deriving correctness proofs for bounded generators, we first need to settle on a
specification. To this end, for each inductive definition for which we derive a
generator, we generate automatically an operator, which we call \lk{iter}, that
maps a natural number to the set of elements that inhabit the inductive relation
and whose proof has height less or equal to the given natural number. This set
will serve as a specification for the bounded generator for a given size
parameter.
This operator has exactly the same shape as the generator, and it is obtained
using the same algorithm; the only thing that changes is that, instead of using
the combinators of the
$\lk{G}~(\lk{option}~-)$ monad, we use those of the \lk{set} monad.
For instance, in the case of the \lk{bst} predicate the \lk{iter} operator looks
like the following:
\iflast\zoe{I don't like how underscore is displayed. The following needs
better typesetting; leaving it for later} \bcp{Are we using the same
font and quoting / anti-quoting conventions here as in the big figure in
section 4? If not, we should at least warn the reader that we are not.}
\leo{This is not ``functions that generate code''. This is a depiction of a Coq code
with a bit of math. Not sure what to warn about exactly}\fi
%
\[\arraycolsep=1pt
\lk{iter\_bst}~in_1~in_2~s =
\left\{
\begin{array}{lr}
\singleton{\lk{Leaf}} & \text{if } s = 0 \\
\singleton{\lk{Leaf}}~ \cup ~ & \text{if } s = s' + 1 \\
\bigcup_{x > in_1}\lk{if}~ x < in_2 ~\lk{then} & \\
{ \quad\quad\quad\,\,\,\,\,\,
\bigcup_{l \in (\lk{iter\_bst}~in_1~x~s')}
\bigcup_{r \in (\lk{iter\_bst}~x~in_2~s')}{\singleton{\lk{Node}~x~l~r}}} & \\
\quad\quad\quad\,
\lk{else}~\emptyset &
\end{array}
\right.
\]
%
The parallels with the generator stand out: we can obtain this by replacing
$\lk{ret}~(\lk{Some}~-)$ with the singleton set (i.e., the return of the \lk{set}
monad),
\lk{bind} with indexed union (i.e., the bind of the \lk{set} monad), and \lk{ret None}
with empty set (i.e., the fail action of the \lk{set} monad).
Using \lk{iter} we can accurately characterize the set of outcomes of a
bounded generator:
%
\[ \isSome \cap \semGen{g~n} = \imset{\lk{Some}}{\lk{iter}~n} \]
%
The proof term for this proposition also has the same structure as the generator,
but this time, instead of monadic combinators, we use the corresponding proof
rules. Since we only care to specify the \lk{Some} part of the set of outcomes
of the generators, we can use slightly modified proof rules that require a
weaker notion of generator monotonicity\iflast\bcp{But then why did we
bother showing the stronger ones?}\fi. In particular, the new rules only
require the generator to be monotonic in the \lk{Some} part of its set of
outcomes. This is captured by the following two definitions.
%
\[ \sizeMonotonicOpt{g} \eqdef \forall s_1~s_2,~s_1 \leq s_2 \rightarrow
\isSome \cap \semGenS{g}{s_1} \subseteq \isSome \cap \semGenS{g}{s_2} \]
%
\[ \boundMonotonicOpt{g} \eqdef \forall s~s_1~s_2,~s_1 \leq s_2 \rightarrow
\isSome \cap \semGenS{g~s_1}{s} \subseteq \isSome \cap \semGenS{g~s_2}{s} \]
%
Using the above definitions we can formulate the alternative proof rules. Below
are examples of reformulated lemmas for \lk{bind} and \lk{size}.
%
\begin{mathpar}
\inferrule*[right=semOptBind]
{\sizeMonotonicOpt{g} \\
\forall x,~\sizeMonotonicOpt{(f~x)} \\
\semGen{g} = s \\
\forall x,~ \isSome \cap \semGen{f~x} = h~x
}
{\isSome \cap \semGen{g~\bindM~f} = \bigcup_{x \in s} {h~x}}
\end{mathpar}
%
%
\begin{mathpar}
\inferrule*[right=semOptSized]
{\boundMonotonicOpt{g} \\
\forall n\in \nat,~\sizeMonotonicOpt{(g~n)}) \\
\forall n,~ \isSome \cap \semGen{g~n} = h~n
}
{\isSome \cap \semGen{\lk{sized}~g} = \bigcup_{n \in \nat} {h~n}}
\end{mathpar}
Our goal is to lift specification from bounded to unbounded generators
using the corresponding lemma for \lk{sized}. To that end, we need a proof that the union of
these sets produced by \lk{iter} over all natural numbers is exactly the set of
elements that satisfy the predicate.
\[ \bigcup_{n \in \nat}{\lk{iter}~n} = P \]
The above proof also requires us to generate a proof that these sets operators
are monotonic in the size parameter.
\[ \forall n_1 ~ n_2,~ n_1 \leq n_2 \rightarrow \lk{iter}~n_1 \subseteq \lk{iter}~n_2 \]
\paragraph*{Monotonicity proofs}
As described above, in order to produce correctness proofs we need to also
produce monotonicity proofs for the unbounded generators. These proofs are used
in both constructing the correctness proofs for bounded combinators, as well as
lifting them to unbounded ones. In order to be able to use the generators as
individual components in other derived generators that come with correctness
proofs, we also lift size monotonicity proofs to unbounded generators. As
in previous cases, this is done using the corresponding lemma for \lk{sized}.
Again, we automate this process by providing the appropriate typeclass
instances. Note that the choice to generate proofs of this weaker notion of
monotonicity is not essential; we could have generated proofs of full
monotonicity instead. However, we opted for this weaker notion as it
significantly simplifies the proof of bound monotonicity.
%% Size monotonicity proofs are also used in the correctness proof in order to be
%% able to use the specification of \lk{semBind} and other combinators.
%% for bounded generators can be lifted to unbounded generators
%% using the corresponding lemmas for lk{sized}.
%% These monotonicity proofs are required to derive the correctness of generators
%% in which the generator is used as a component.
% reference sized
% first monotonicity
% then iter + final proof
%% In particular, first we generate proofs for
%% the bounded generators, and then we automatically lift the specifications to
%% unbounded generators using the lemmas provided for {\tt size}.
%% This proof uses the lemmas that require generators to be size monotonic, hence
%% we should make sure that each individual generator used is size monotonic.
%% QuickChick built-in generators come with a monotonicity proof and, in addition,
%% for each derived unbounded generator we can obtain this proof by using the
%% corresponding lemma for \lk{size} and the monotonicity proofs for the unbounded
%% generator. These cover all the cases of generators used by the automatically
%% derive generators and hence the correctness proof will always go through.
%% We pack these definitions in a type class and we automatically
%% derive instances of this typeclass for each inductive predicate.
%* BoundMonotonicity hard for Nones, but we only care about none
%* Then we do not obtain SizeMonotonicity for the top-levels
%* It's OK -> only reason about the some part so this is enough
%% \zoe{The rest in this subsection are not up-to-date }
%% The first proof component that we need is that the automatically derived
%% generators are monotonic in the size parameter. This will allow us to use the
%% lemmas that require generators to be monotonic, . The proof term is easy to generate using the lemmas provided by
%% QuickChick. \zoe{Show the parameters to the algorithm to give an example of how
%% we generates proofs} \zoe{Mention that this is monotonicity for Some only?}
%% Using this typeclass we can express the correctness of sized generators. In
%% particular we require that for a given size parameter the all elements in the
%% set for this parameter can be generated (completeness) and that everything that
%% is generated is either some element in the set or {\tt None} (soundness).
%% %
%% \verifcode{genSizedCorr}
%% %
%% Notice that the set of outcomes of the generator might or might not include {\tt
%% None} and we leave this unspecified. This is because generation can fail when
%% the constraints for a path are not satisfiable and backtracking is required.
%% The last required proof in order to be able to obtain the correctness proof of
%% the \fix{unsized} generators is that the sized generator is monotonic in its
%% size parameter. This will allow us to obtain both a proof that the \fix{unsized}
%% generator is correct and that it is size monotonic using the corresponding
%% lemmas for {\tt sized}. The later proof is important because this generator
%% might be used in other constructions that will require monotonicity of this
%% generator to be proved correct.
\subsection{Typeclasses for Proof Generation}\label{subsec:verif_typeclasses}
As we did for generators in Section~\ref{sec:algorithm}, we rely on
typeclasses to connect individual proof components and lift specifications
to unbounded generators. In this subsection we describe the extensions to
the typeclass infrastructure of QuickChick presented in Section~\ref{sec:quickchick}, which are needed in order to
achieve this. We use Coq notation so that we can display the actual
typeclass definitions; the notation \verb|:&:| denotes set intersection and
the notation \verb|@:| the image of a function over some set.
%% So far we have generated every proof component about the bounded
%% generators that is necessary to obtain the proofs of monotonicity and correctness for the
%% unbounded generators. In what we have presented above this last step is not
%% automated and it leaves to the user the application of \lk{sized} lemmas in
%% order to obtain the top-level proofs. To automate this process, which is
%% essential as other derived proofs may rely on these top-level proofs, we extend
%% the typeclass infrastructure of QuickChick so that these proofs are
%% automatically derived by instance resolution. Typeclass resolution also allows
%% us to uniformly handle the abstract type parameters of an inductive definition
%% as well as the \fix{arguments to the constructors}. In the rest of the section
%% we will present the extensions in the typeclass hierarchy of QuickChick that
%% allow to fully automate the process of deriving correct generators for inductive
%% relations as well as the corresponding proofs.
%% which makes the dependencies between generated proofs explicit.
%% First, we introduce typeclasses to represent bounded and unbounded generators
%% that satisfy a certain predicate.
%% %
%% \qccode{src/DependentClasses.v}{genSTSized_class}
%% %
%% \qccode{src/DependentClasses.v}{genST_class}
%% %
%% These classes, of course, provide no guarantee that the provided generator is
%% correct, but, as \lk{GenSized} and \lk{Gen}, they allow us to give a uniform
%% interface to generators that satisfy certain predicate. We also add an instance
%% to convert automatically \lk{GenSizedSuchThat} inctances to \lk{GenSuchThat}
%% instances using \lk{sized}.
%% %
%% \qccode{src/DependentClasses.v}{GenSuchThatOfBounded}
%% %
%% Bounded generators are generated automatically using the algorithm we propose in
%% this paper. Unbounded generators are generated automatically by the above
%% instance.
\paragraph*{Monotonicity}
First we extend the typeclass hierarchy to encode size and bound monotonicity
properties.
\verifcode{SizeMonotonicOpt}
\verifcode{BoundMonotonicOpt}
We automatically generate proofs that derived bounded generators are bound and
size monotonic by explicitly constructing the proof term and we automatically
create instances of these classes. Size monotonicity can then be derived for
unbounded generators using the following provided instance.
\verifcode{SizeMonotonicOptOfBounded}
Given a \lk{GenSizedSuchThat} instance for a predicate $P$ (\lk{H1} above),
which provides access to a constrainted bounded generator \lk{arbitrarySizeST
P}, and instances of size and bound monotonicity for this generator (\lk{H2}
and \lk{H3}), we can obtain an instance of size monotonicity for unbounded
generator for this predicate, \lk{arbitraryST P}, which is also
obtained automatically by the corresponding instance.
\paragraph*{Set Operators}
To express the correctness property of generators we introduce a typeclass that gives
a generic interface to predicates which are equipped with an
\lk{iter} operator.
%
\verifcode{sizeEqs}
%
\paragraph*{Correctness}
We can define a subclass of the above class, which is used to characterize
bounded generators that are correct with respect to a predicate.
%
\verifcode{genSizedCorr}
%
In the above, we are requiring that $P$ is an instance of the \lk{Iter} class
in order to be able to use \lk{iter} to express the correctness property.
Following our usual practice, we also define a class for correct unbounded
generators.
%
\verifcode{SuchThatCorrect}
%
%We also extend QuickChick with two other typeclasses \lk{sizeMonotonicOpt} and
%\lk{boundMonotonicOpt} whose definition is straightforward and hence omitted.
%\leo{I can picture what they are. But I know the code. I'll defer to Benjamin as to
%how straightforward this is :)}\bcp{Benjamin is completely lost.}
%%
%
As before, we automatically generate instances for correctness of bounded
generators by proving the proof terms, and we then lift them to unbounded
generators by adding the corresponding instance.
%The instance for monotonicity is similar.
\verifcode{SuchThatCorrectOfBounded}
%
The above instance is similar to the one for monotonicity but it additionally
requires an instance for correctness of the unbounded generator (\lk{H5}). It
also requires an instance of the \lk{Iter} class for \lk{P} (\lk{H2}). This
instance is required as an (implicit) argument to the instance of
correctness and also in the proof itself as it provides the specification
of \lk{iter}.
%
%% When we construct proof terms we assume that the generators used by the derived
%% generators are instances of the corresponding typeclasses. For instance, when we
%% construct correctness proofs, we assume that the generators used are both
%% correct and monotonic. This is guaranteed for all the built-in generators of
%% QuickChick, as well as the automatically derived generators. However, custom
%% generators can be plugged in the derived generators as long as the required
%% typeclass instances are provided.
\iflast\zoe{TODO add discussion about abstract type parameters}\fi
%% We rely on typeclass resolution not only to automatically obtain unbounded
%% generators and the corresponding proofs, but also tosimplify the proof
%% generation itself by providing a generic interfaces for the various proof
%% components. For instance, the correctness proof require that all the individual
%% generators used are monotonic and correct. This goals are automatically resolved
%% by typeclass resolution since all of the individual generators used by the
%% derived ones come with such inctances. In particular, QuickChick provides such
%% proofs for each built-it generator and every automatically derived unbounded
%% generator we can derive correctness and monotonicity inctances.
%% We can also abstractly handle type parameters of inductive relations by adding
%% the nessecary typeclass constraints as parameters of the generated instance. For
%% example, when deriving generators for a relation with a type parameter we will
%% add a constraint that the parameter has an instance of \lk{Gen} and when we
%% derive the corresponding correctness proof we will also require monotonicity and
%% correctness instances for the type parameter.
%% In particular for abstract types we assume that they are instances of the
%% requires typeclass while for concrete types we are requiring that this instances
%% are already in the environment.
% \begin{itemize}
% \item Motivation: auto-generated code. Guarantees?
% \item Overview of QuickChick's proving framework
% \item Proof generation (monotonicity?) How much detail?
% \end{itemize}
\section{Implementation}
\label{sec:impl}
Our initial implementation of the generator derivation algorithm
interfaced directly with Coq's internals. But, even for the
simply-typed inductive
generators of Section~\ref{sec:quickchick}, this was neither an extensible nor a
maintainable approach. Coq's term data structure, for example, contains
far too much type information that our application does not care about. Similarly,
the internal functions that produce Coq expressions take more arguments that we
need, in order to accurately populate the rich data structure.
To facilitate deriving generators and proof terms (as well as other needed
infrastructure such as \lk{Show} instances and shrinking functions that we
haven't touched on in this paper), we wrote a small generic programming
framework consisting of two parts: a high level representation of the class
of inductive terms we target and a small DSL for producing Coq expressions.
%
We represent the class we target with the following
datatype:
%lst
%\pagebreak
\begin{lstlisting}[basicstyle=\ttfamily\small, basewidth=.48em]
type dep_type =
| DArrow of dep_type * dep_type (* Unnamed arrows *)
| DProd of (var * dep_type) * dep_type (* Binding arrows *)
| DTyParam of ty_param (* Type parameters *)
| DTyCtr of ty_ctr * dep_type list (* Type Constructor *)
| DCtr of constructor * dep_type list (* Data Constructor *)
| DTyVar of var (* Use of a type variable *)
\end{lstlisting}
%
The representation is relatively standard. Note that arrows and products are
treated as a top-level constructors, since they are of
particular importance: arrows can be used to represent side-conditions and
products to capture the universally quantified variables of each
constructor. Each type above (like \lk{var} or \lk{constructor}) is
an opaque wrapper around Coq identifiers, completing the separation of the
generic library user from Coq internals.
For our term-building DSL, we provide higher-order abstract syntax
combinators. To guarantee well-scopedness of generated terms, we handle fresh
name generation internally. For example, the combinator for a local recursive
fixpoint \lk{let fix ... in} (like the one used to define \lk{aux\_arb} in the
previous sections) is:
%
\begin{inlinecode}
val gRecIn : string -> string list -> (var * var list -> coq_expr) ->
(var -> coq_expr) -> coq_expr
\end{inlinecode}
%
To construct a fixpoint, we require a \lk{string}, the base name for the
recursive fixpoint, and a list of strings, the base name for each
argument. Internally, we convert each such string to a fresh Coq
identifier. These identifiers can be used to construct other Coq
expressions: inside the fixpoint we can use the opaque representation of both
the fixpoint name and its arguments; in the continuation we can only use the
fixpoint itself.
% \subsection{Typeclasses for Automated \lk{Prop} Testing}
% \bcp{Not sure I see how this is related to the main contributions of the
% paper. If it's a side discussion, better leave it for someplace else and
% use our space on explaining the main algorithm.}
% \leo{Using typeclasses/proofs to automatically test Props?}
% \leo{Things I would like to say}
% \begin{itemize}
% \item One attractive feature of Haskell's QuickCheck is the use of
% typeclasses to automate testing
% \item Explain function typeclass instance
% \item Explain prop typeclass instance of checkable with Dec
% \item explain using conditional generators automatically
% \end{itemize}
\section{Evaluation}
\label{sec:eval}
We evaluate two aspects of our generators: the applicability of the restricted
class of inductive types we target~(\ref{sec:eval-sf}) and the efficiency of the
derived generators compared to handwritten ones~(\ref{sec:eval-ifc}).
\subsection{QuickChecking Software Foundations}
\label{sec:eval-sf}
To evaluate the applicability of our algorithm we tried to automatically test a
large body of specifications that are representative of those commonly used in
verifying properties of programming languages. Such a body of specifications
can be found in Software Foundations~\cite{SF}, a machine-checked textbook for
programming language theory and verification using Coq. We attempted to
automatically test every theorem or lemma in the suggested main course of the
textbook, all the way through the simply typed lambda calculus chapters.
Our findings are summarized in Figure~\ref{fig:range-graph}.
%
To avoid skewing our findings, we separately count certain classes of
examples.
%
Of the 232 nontrivial (non-unit-test) theorems we considered, 194 (84\%) are
directly amenable to PBRT; the 38 remaining theorems deal with generation
for higher-order properties, which we deem too difficult for automatic
test-case generation (we give examples below). Of the 194 theorems we
believed ``should be testable,'' 160 (83\%) could be tested using our
implemented algorithm. This demonstrates that the class of inductive
propositions targeted by our narrowing generalization is broad enough to
tackle many practical cases in the \SF{} setting. The rest of this section
discusses the different classes of theorems we considered and our
methodology for testing each one.
\begin{figure}
\vspace{-2em}
\centering
\includegraphics[scale=0.42]{LineRangeGraph.png}
\vspace{-2em}
\caption{Evaluation Results}
\label{fig:range-graph}
\end{figure}
\iflast\bcp{Another experiment we could do, if there was time: build all the
generators completely by hand for everything, and see how many lines of
code that takes (i.e., how many lines of code are saved by using our
generator generator).}\leo{Acknowledgment for reviewers?}\fi
First of all, \SF{} incorporates a large number of unit tests,
like the following test for disjunction:
\begin{inlinecode}
Example test_orb1: (orb true false) = true.
\end{inlinecode}
Such examples are trivially checkable and unininteresting from a generation
perspective.
On the other hand, a class of lemmas that are completely out of scope of
generation techniques deal with universally quantified higher-order
properties. Consider the following canonical example of a Hoare
triple:
\begin{inlinecode}
Theorem hoare_seq : forall P Q R c1 c2,
{{Q}} c2 {{R}} -> {{P}} c1 {{Q}} -> {{P}} c1;;c2 {{R}}.
\end{inlinecode}
Testing such a property would require generating arbitrary elements of
type \lk{state -> Prop}, which is beyond current automatic random generation
techniques.
%
% \bcp{Not clear to me that you couldn't do something simple
% that would work (e.g., generate simple boolean / arithmetic formulas over
% a small set of variables). But it's pretty clear that you wouldn't expect
% to be able to do this automatically. (More generally, we should clarify
% early on that our goal is not to generate all conceivable generators
% automatically, but just to remove the tedium of writing simple ones.)}
% \leo{Added ``automatic''}
%%
For context, the number of higher-order properties we excluded were 38; 36
of them came from the Logic and Hoare logic chapters that heavily use
quantification over \lk{Prop}s.
Finally, a third class of properties that could be interesting from a generation
perspective but are a poor fit for property-based random testing are existential
properties. For example, consider progress for a type system:
\begin{inlinecode}
Conjecture progress : forall t T, |- t \in T -> value t \/ exists t', t ===> t'.
\end{inlinecode}
While generating \lk{t} and \lk{T} such that \lk{t} has type \lk{T} is both
interesting and possible within the extension of QuickChick presented in this
paper, it is not possible to decide whether the conclusion of the property
holds! However, most of the time, it is possible to rewrite existential
conclusions into decidable ones. For example, for a deterministic step relation,
we could write a partial \lk{step} function and rewrite the conclusion to check
whether the term \lk{t} can actually take a step: \lk{isSome (step t)}.
With the above in mind we proceeded to automatically derive generators for all
simple inductive types, generators for different modes for inductive relations,
as well as proofs for both. We completely elided unit tests, counted (but
otherwise ignored) properties that required generation of higher order
properties, and converted conclusions to decidable when necessary.
%
We then turned each property into a \lk{Conjecture}---an automatically admitted
property---and attempted to test it with QuickChick. For
example, the preservation property became:
\begin{inlinecode}
Conjecture preservation : forall t t' T, |- t \in T -> t ===> t' -> |- t' \in T.
QuickChick preservation.
\end{inlinecode}
This simulates a common workflow in interactions with the Coq proof assistant:
in order to prove a large theorem (e.g. type safety), provers often \lk{Admit}
smaller lemmas to construct the proof, discharging them afterwards. However,
admitting a lemma that is too strong can lead to a lot of wasted effort and
frustration\iflast\leo{I would like to cite something like Sam's thesis here. do we
have something for that?}\fi. Using QuickChick, users can uncover bugs early on while
building confidence in such conjectures.
For a small portion of the theorems we allowed minor changes (i.e., converting
preconditions like \lk{beq\_nat n1 n2} to \lk{n1 = n2}). Overall, we only
performed one major change: converting Software Foundation \lk{Map}s from a
functional representation to an explicit list-based one. A functional
representation for maps is convenient for looking up element's associations,
but---unless the domain of the function is bounded---makes it completely
impossible to do the reverse. That requirement is very common in
generation---for instance, picking a variable with a specific type from a given
context. Moreover, a lot of properties needed to decide equivalence of two
maps, which is also impossible in a functional
representation. Therefore, we changed maps to a more generation-friendly
variant. The new map code was similar in length with respect to the old one ($\sim 40$
lines), including automatic derivations of generators and decidability
instances, but resulted in many syntactic changes across the rest of the
chapters.
\subsection{QuickChecking Noninterference}
\label{sec:eval-ifc}
\leo{This entire section needs a lot of rewriting}
To evaluate the efficiency of our approach, we conducted a case study comparing
the runtime performance of our derived generators against carefully tuned
handwritten
ones:
the generators for the information-flow control experiments in
\citet{TestingNI, testing_ni_jfp},
which generate indistinguishable pairs of machine states using Haskell's
QuickCheck to discover noninterference violations. These generators had already
been ported to QuickChick, where they formed the main case study for the proof
generation framework of~\citet{itp2015}. There, they were systematically
evaluated using a rigorous mutation testing methodology, which we reused here to
ensure that our derived generators had roughly the same bug-finding
capabilities. Our experiments showed that the derived generators were
$1.75\times$ slower than the corresponding handwritten ones, while producing the
same distribution and bugfinding performance.
In more detail: Dynamic information-flow control\cn tags data values with
security
levels, called {\em labels}, and uses them to prevent flows from {\em high}
(secret) to {\em low} (public) data. In particular, \citet{TestingNI}
enhanced a
simple, low-level stack machine with label information and tested it for {\em
termination-insensitive noninterference}: given two {\em indistinguishable}
machine states, i.e. states that differ only in high data, running them to
completion should yield indistinguishable states. This is a prototypical
example of a
conditional property: if we were to generate pairs of arbitrary machine
states and
discard those that are not indistinguishable, we would almost never exercise the
conclusion! Instead, \citet{TestingNI} generated a single arbitrary machine state
first and then {\em varied} that state to produce a new one that was
indistinguishable (by construction).
For our evaluation, we focused on a stronger property (also considered by \citet{TestingNI}), {\em single-step
noninterference}, which only runs both machines for a single step. As
\citet{TestingNI} showed, this makes generators for valid initial states
substantially simpler: since only one instruction will be executed, memories
do not need to be longer than two elements (no more than one element can be
accessed by each machine), integer values that are valid pointers are only 0
or 1 (since the
memories are two elements long), and stacks do not need to be large either.
Consider, for instance, a generator for stacks (of a given length \lk{n}),
which can be empty (\lk{Mty}), cells that store a tagged integer
(\lk{Cons}), or specially marked stack frames that store a program counter
to be used by a future \lk{Return} instruction (\lk{RetCons});
\lk{gen\_atom} produces mostly in-bounds tagged integers.
%
\begin{inlinecode}
Fixpoint gen_stack (n : nat) : G Stack :=
match n with
| O => returnGen Mty
| S n' =>
freq [ (10, liftGen2 Cons gen_atom (gen_stack n'))
; (4, liftGen2 RetCons gen_atom (gen_stack n')) ]
end.
\end{inlinecode}
%
The behavior of this generator can be described by a simple inductive
predicate,
where \lk{good\_atom} describes the behavior of \lk{gen\_atom}.
%
\begin{inlinecode}
Inductive good_stack : nat -> Stack -> Prop :=
| GoodStackMty : good_stack 0 Mty
| GoodStackCons : forall n a s ,
good_atom a -> good_stack n s -> good_stack (S n) (a :: s)
| GoodStackRet : forall n pc s,
good_atom pc -> good_stack n s -> good_stack (S n) (RetCons pc s).
\end{inlinecode}
%
Finally, we can achieve the same distribution with a weight annotation before
deriving generators.
%
\begin{inlinecode}
QuickChickWeights [(GoodStackCons, 10); (GoodStackRet, 4)].
Derive ArbitrarySizedSuchThat for (fun s => good_stack n s).
\end{inlinecode}
The implicit assumptions for single-state generators are encoded in inductive
predicates, and the indistinguishability relation is used to derive variation
generators\bcp{?}.
We tested the single-step noninterference property 10000 times using both the
handwritten and the derived generators. Our derived generators were
1.75$\times$ slower than the handwritten ones, while both generators uncovered
all mutants successfully. To ensure both generators yield similar distributions
of inputs, we used QuickChick's \lk{collect} to determine the number of times
each instruction was generated during those 10000 tests (as this was the metric
that was used to fine-tune the handwritten generators in the first place).
The observed 1.75$\times$ slowdown is mostly due to the added overhead of local
backtracking and extraneous matches like the one in the \lk{goodTree} example of
Section~\ref{sec:example}. A few local optimizations (like pulling a match
outside of a call to \lk{backtrack}) could further improve on our performance,
but would require additional work to produce the corresponding proof terms.
Still, this overhead is much better than the order-of-magnitude
overhead of interpreted approaches like Luck~\cite{LuckPOPL}.
%
Finally, what we gain in return for this loss in performance is that the declarative
nature of the inductive predicates exposes exactly what assumptions are made
about the generated domain, while the produced proofs guarantee completeness for
that domain.
\section{Related Work}
\label{sec:related}
\paragraph*{Narrowing-based approaches}
The most closely related works\leo{can we say works?}, narrowing-based approaches, have
already been briefly discussed in the introduction. Such approaches have given
rise to tools for many languages, including Haskell~\cite{ClaessenFLOPS14} and
Racket~\cite{FetscherCPHF15}, as well as domain-specific languages, like
UDITA~\cite{GligoricGJKKM10} and Luck~\cite{LuckPOPL}. All of these artifacts
successfully adapt variants of narrowing to generate values satisfying
preconditions.\iflast\leo{with Racket and Luck also incorporating constraint-solving elements.}\fi
%
In this paper, we build upon their success, adapting narrowing for Coq's
inductive relations, showing how to produce Coq generators getting rid of
interpretation overheads, and producing proofs of the generators correctness in
the process.
\paragraph*{Smart enumeration approaches}
Another closely related line of work deals with {\em enumeration} for data
satisfying invariants. Perhaps closest to us is the work in the context of
Isabelle's QuickCheck~\cite{Bulwahn12, Bulwahn12smartgen}. There, Bulwahn
targets a similar simply-typed subset of Isabelle producing ML
enumerators.
\iflast\leo{Focus more on incomplete mode analysis? Will it be too aggressive?}\fi
%
A different enumeration approach, based on laziness, is taken for Haskell's Lazy
SmallCheck~\cite{RuncimanNL08}, as well as Scala's
SciFe~\cite{KurajKJ15}. Laziness conceptually mimics narrowing by delaying the
instantiation of variables when possible.
%
On the other hand, \cite{FischerK07} and ~\cite{ChristiansenF08} leverage the
built-in narrowing mechanism of the functional logic programming language
Curry~\cite{Curry} to enumerate data satisfying invariants.
%
While enumeration based testing can be very successful in various domains, we
turn to random testing as the complexity of our target applications---like, for
example, well typed lambda terms---makes enumeration intractable.
\paragraph*{SMT-based approaches}
An alternative approach to generating inputs satisfying a precondition $P$ is to
translate $P$ into logic and then use an SMT solver. Such a translation has been
performed a few times~\cite{SeidelVJ15, CarlierDG13, GotliebICST09}). The most recent and
efficient one, Target~\cite{SeidelVJ15}, targets Liquid Haskell preconditions in
the form of refinement types. While Target outperforms Lazy SmallCheck and
similar tools in a lot of cases, Luck~\cite{LuckPOPL} shows that narrowing can
still perform better on occasion. Moreover, the complexity of the translation
leaves little room, if any, for controlling the distribution of generated
inputs, unlike in QuickChick-derived generators where users can leverage
Luck-style annotations to control {\tt backtrack} weights.
\leo{Zoe: Do you want to add a paragraph about related work in proof term generation?
We say nothing about that...}\bcp{What kind of papers are you thinking of?
The basic idea of generating proof terms seems very broad---I'm not sure
what I'd cite. But is there something more specific?}
\paragraph*{Inductive to Executable Specifications}
\iflast
\leo{This is less related, but should be here. Not sure if it should go before or after the
proof related stuff by zoe}
\fi{}
At a high level, the algorithm described in Section~\ref{sec:algorithm} has
similarities to earlier attempts at extracting executable specifications from
inductive ones~\cite{DelahayeDE07,TollitteDD12} in the Coq proof assistant. In
principle, we could use the algorithm described in this section to obtain a
similar transformation. Consider for example, an inductive predicate \lk{P : A
-> B -> C -> Prop}. If we transform it to a predicate \lk{P' : A -> B -> C ->
unit -> Prop} by adding \lk{()} as an additional argument at every occurrence
of \lk{P}, we could ask our algorithm to generate \lk{x} such that
\lk{P' a b c x} holds for all \lk{a}, \lk{b}, and \lk{c}. We would then
essentially obtain a partial decision procedure for \lk{P}, based on whether
the generator returns \lk{Some} or \lk{None}. In fact, our algorithm can be seen
as a generalization of their approach as the derived decision procedures are
equivalent (modulo size) for the class of inductive datatypes they handle that yields
deterministic functional programs.
\section{Conclusion and future work}
\label{sec:concl}
We have presented a narrowing-based algorithm for compiling
dependently-typed inductive relations into generators for random data
structures satisfying these relations, together with correctness proofs. We
implemented it in the Coq proof assistant and evaluated its applicability by
automatically deriving generators to test the majority of theorems in
\SF{}.
In the future, we aim to extend our algorithm to a larger class of inductive
definitions. For example, incorporating function symbols is straightforward:
%
simply treat functions as black boxes, instantiating all
of their arguments before treating the result as a $\fixed$ range.
%
For statically known functions, we could also leverage Coq's open term reduction
to try to simplify function calls into constructor terms.
%
Finally, it would be possible to adapt the established narrowing approaches for
functional programs to meaningfully instantiate unknown function arguments
against a known result pattern, just like in Luck~\cite{LuckPOPL}.
We also want to see if our algorithm can be adapted to derive decidability
instances for specifications in \lk{Prop}, allowing for immediate, fully
automatic testing feedback. We are also interested in \emph{shrinkers} for
constrained data, to complete the property-based testing ecosystem for Coq.
%% Acknowledgments
\begin{acks} %% acks environment is optional
%% contents suppressed with 'anonymous'
%% Commands \grantsponsor{}{}{} and
%% \grantnum[]{}{} should be used to
%% acknowledge financial support and will be used by metadata
%% extraction tools.
We are grateful to
%
Maxime D\'en\`es,
C\u{a}t\u{a}lin Hri\c{t}cu,
John Hughes,
George Karachalias,
Micha{\l} Pa{\l}ka,
Antal Spector-Zabusky,
the CLA workshop community,
and the Penn PLClub
for their useful comments.
This material is based upon work supported by the
\grantsponsor{GS100000001}{National Science
Foundation}{http://dx.doi.org/10.13039/100000001} under Grant
No.~\grantnum{GS100000001}{1421243} ({\em Random Testing for Language Design}),
Grant No.~\grantnum{GS100000001}{1521523} ({\em Expeditions in Computing: The
Science of Deep Specification}), and Grant No.~\grantnum{GS100000001}{1407794}
({\em Optimizing Compilation of Dependently Typed Languages}). Any opinions,
findings, and conclusions or recommendations expressed in this material are
those of the author and do not necessarily reflect the views of the National
Science Foundation.
% Antal
% Zoe: I should mention that I was funded by the CertiCoq grant
\end{acks}
%% Bibliography
%\bibliography{bibfile}
\bibliography{../../quick-chick,local}
%% Appendix
%\appendix
%\section{Appendix}
%
%Text of appendix \ldots
\end{document}