Watching a macro as it expands

;; macros are code transformers
;; in a lisp, that means tree transformers
;; let's look at the -> macro, built into clojure.

(use 'clojure.contrib.repl-utils)
(source ->)

;; (defmacro ->
;;   "Threads the expr through the forms. Inserts x as the
;;   second item in the first form, making a list of it if it is not a
;;   list already. If there are more forms, inserts the first form as the
;;   second item in second form, etc."
;;   ([x] x)
;;   ([x form] (if (seq? form)
;;               (with-meta `(~(first form) ~x ~@(next form)) (meta form))
;;               (list form x)))
;;   ([x form & more] `(-> (-> ~x ~form) ~@more)))



;; What's 20C in Fahrenheit?
;; Multiply by nine, divide by 5, add 32
(-> 20 (* 9) (/ 5) (+ 32))

;; let's look at the tree for that
(use 'clojure.inspector)
(inspect-tree '(-> 20 (* 9) (/ 5) (+ 32)))

;; What does that look like after macroexpansion?
(use 'clojure.walk)
(macroexpand-all '(-> 20 (* 9) (/ 5) (+ 32)))
(inspect-tree (macroexpand-all '(-> 20 (* 9) (/ 5) (+ 32))))


;; How does the threading macro work?
(macroexpand-1 '(-> 20 (* 9) (/ 5) (+ 32)))

;; We can follow the recursion by using macroexpand-1 on each subexpression in turn.

;;slightly tidied up, stages are:

(macroexpand-1 '(-> 20 (* 9) (/ 5) (+ 32)))

(macroexpand-1
'(-> 
  (-> 20 (* 9))
  (/ 5) (+ 32)))

(macroexpand-1
 '(-> 
   (-> 
    (-> 20 (* 9)) 
    (/ 5)) 
   (+ 32)))

(macroexpand-1
 '(+ 
   (-> 
    (-> 20 (* 9)) 
    (/ 5))
   32))

`(+ 
  ~(macroexpand-1 '
    (-> 
     (-> 20 (* 9)) 
     (/ 5)))
  32)

`(+ 
  (/ 
   ~(macroexpand-1 '
     (-> 20 (* 9)))
   5)
  32)

(+ 
 (/ 
  (* 20 9) 
  5) 
 32)

;;This is a bit long-winded, how would we do this automatically?

(defn slow-tree-transformer [fn tree]
  (cond (not (seq? tree)) (fn tree)
        (empty? tree) '()
        :else (let [nxt (fn tree)]
                (if (not (= nxt tree)) nxt
                    (let [nxt (fn (first tree))]
                      (if (= nxt (first tree))
                        (let [nxt (slow-tree-transformer fn (first tree))]
                          (if (= nxt (first tree))
                            (cons (first tree) (slow-tree-transformer fn (rest tree)))
                            (cons nxt (rest tree))))
                        (cons nxt (rest tree))))))))


(def f (fn[x] (slow-tree-transformer macroexpand-1 x)))

(defn iterate-to-stable [fn start]
  (let [next (fn start)]
    (if (= next start) (list start)
        (cons start (iterate-to-stable fn next)))))

(use 'clojure.contrib.pprint)
(println "-----------------------------------")
(doall (map (fn [x] (pprint x) (println)) (iterate-to-stable f '(-> 20 (* 9) (/ 5) (+ 32)))))
(println "-----------------------------------")

Jihad Scanner

There's a very promising looking new lisp book out, Doug Hoyte's "Let over Lambda" http://letoverlambda.com/

My copy arrived this morning, and to keep me honest while reading it, I wondered how much of it could be translated into clojure. Here's the first example I found interesting:
 

;; The jihad scanner from let over lambda. Watch a communications channel for a word.

(defn make-scanner [ watchword ]
  (let [ curr (atom watchword) ]
    (fn [ chunk ]
      (doseq [c chunk]
        (if @curr
          (swap! curr (fn [cv] (cond (= (first cv) c)        (next cv) 
                                     (= (first watchword) c) (next watchword)
                                     :else                    watchword)))))
      (nil? @curr))))


;; I've added a clause so that jjihad still fires the detector. I think the original version doesn't catch jjihad
;; but does catch qjihad, which seems inconsistent.

;; Compared to the Common Lisp version, the requirement to wrap mutable state in an atom has cost us
;; a couple of spurious @s, and required "(swap! curr (fn [cv]" instead of "(setq" without really 
;; buying us anything. 

;; On the other hand, the fact that strings are just another form of seq has simplified the function
;; and generalised it to an arbitrary subsequence detector.


(let [a (make-scanner "jihad")]
  (a "We will start")
  (a "the jih")
  (a "ad tomorrow"))


(let [a (make-scanner "jihad")]
  (a "The jijihad starts tomorrow"))

(let [a (make-scanner "jihad")]
  (a "The jij ihad starts tomorrow"))

(let [a (make-scanner "jihad")]
  (a "Don't mention the j-word"))

(let [a (make-scanner '(97 98 99))]
  (for [x (partition 3 (range 110))] (a x)))

(let [a (make-scanner '[a b c])]
  (for [ x '((a b) (c d)) ] (a x)))

(let [a (make-scanner '[a b c])]
  (for [ x '((a b) [c d]) ] (a x)))

Requiring all possible namespaces

It often irritates me that I can't use clojure's built in documentation unless I've already loaded the library I need.

Sometimes it turns out that the generic-looking function I need already exists, but it can be hard to find it. Sometimes I can remember the function I want, but not its precise name or library namespace.

It's also often the case that the best way to understand a function is to read the source.

The other day it occurred to me that a function to find all the namespaces available on the classpath might help quite a bit, and I started to write one.

And at that point I had the obvious recursive thought.

   

;; Seek and ye shall find. Thank you Stuart Sierra.
(use 'clojure.contrib.find-namespaces)

;; First let's see what our classpath is:
(println "Classpath")
(doseq [p (seq (.getURLs (java.lang.ClassLoader/getSystemClassLoader)))] (println (str p)))


;; In my case, that's the clojure.jar and clojure.contrib.jar files, and also the directory
;; containing the files for swank-clojure, the library which allows it to talk to emacs.
(println "Swank namespaces")
(println (for [s (for [n (all-ns)] (name (ns-name n))) :when (. s contains "swank")] s))


;; Because I'm still a beginner, and not brave enough to modify swank, I'm more interested in 
;; the namespaces provided by clojure and clojure.contrib

;; So far, the following have managed to get themselves loaded:
(println "None-swank namespaces")
(use 'clojure.contrib.pprint)
(pprint (sort (for [s (for [n (all-ns)] (name (ns-name n))) :when (not (. s contains "swank"))] s)))


;; But the following are in fact available
(pprint (find-namespaces-on-classpath))

;; Again, let's ignore the swank-related ones.
(pprint (filter #(not (. (str %) startsWith "swank")) (find-namespaces-on-classpath)))


;;For me there are only namespaces starting with either swank or clojure, because I don't have much
;;of a classpath. But it's easy enough to check that.
(defn find-namespaces-starting-with [strng]
  (filter #(. (str %) startsWith strng) (find-namespaces-on-classpath)))

(map count (map find-namespaces-starting-with '("clojure" "swank" ""))) ; should add up


;; Some of the clojure.contrib libraries fail to load for me, so we have to bullet-proof require
(defn require-may-fail [ns]
  (try
   (print "Attempting to require " ns ": ")
   (require ns)
   (println "success")
   (catch Exception e (println "couldn't require " ns "\nException\n" e "\n\n"))))


;; Now we can load everything
(doall (map require-may-fail (filter #(. (str %) startsWith "clojure") (find-namespaces-on-classpath))))


;; Now we have lots and lots of namespaces available: The number of 'batteries included' is impressive.
(println "None-swank namespaces")
(pprint (sort (for [s (for [n (all-ns)] (name (ns-name n))) :when (not (. s contains "swank"))] s)))


;; And the find-doc, doc, and source commands become really useful
;; Of course we not only have to find source, we actually can, now.
(find-doc "source code")

;; Also note that all this requiring hasn't brought any new functions into play yet.
;; To use something without qualification, you need to refer as well.
(refer 'clojure.contrib.repl-utils)
(doc source)
(source source)
(source get-source)
(:file (meta (resolve 'source)))

Other sources of enjoyment are the find and grep commands, and browsing clojure.contrib.jar and clojure.jar . These work particularly nicely under emacs. A good arrangement of find and grep is (assuming that your clojure sources are under ~/opt):

find ~/opt/ -name "*.clj" -and -type f -print0 | xargs -0 -e grep -nH -e "repl-util"

Using the trace function in clojure.contrib

There's a fine trace macro in clojure.contrib.trace, which really can help when looking at recursive functions.
Here's a very simple example with our dear friend 'naive fibonacci'.

   
user> (defn fib [n]
        (if (< n 2) n
            (+ (fib (- n 1)) (fib (- n 2)))))
#'user/fib
user> (use 'clojure.contrib.trace)
nil
user> (dotrace (fib) (fib 3))

TRACE t1880: (fib 3)
TRACE t1881: |    (fib 2)
TRACE t1882: |    |    (fib 1)
TRACE t1882: |    |    => 1
TRACE t1883: |    |    (fib 0)
TRACE t1883: |    |    => 0
TRACE t1881: |    => 1
TRACE t1884: |    (fib 1)
TRACE t1884: |    => 1
TRACE t1880: => 2
2
user>