From af5f03424fe74f2684c8f57f47243c678b9db5c8 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Tue, 13 Jun 2023 14:21:52 -0700
Subject: [PATCH 01/22] Fix #240: Speed up set-difference and friends

Use a hashtable to speed up set-difference when the test is suitable
and the lists are long enough.
---
 src/code/list.lisp | 50 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 8 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 590564a46..21bd6531e 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -813,18 +813,52 @@
     res))
 
 (defun set-difference (list1 list2 &key key
-			     (test #'eql testp) (test-not nil notp))
+				     (test #'eql testp) (test-not nil notp))
   "Returns the elements of list1 which are not in list2."
   (declare (inline member))
   (if (and testp notp)
       (error "Test and test-not both supplied."))
-  (if (null list2)
-      list1
-      (let ((res nil))
-	(dolist (elt list1)
-	  (if (not (with-set-keys (member (apply-key key elt) list2)))
-	      (push elt res)))
-	res)))
+  (flet ((default-impl (list1 list2)
+           (if (null list2)
+               list1
+               (let ((res nil))
+		 (dolist (elt list1)
+                   (if (not (with-set-keys (member (apply-key key elt) list2)))
+                       (push elt res)))
+		 res))))
+    (cond ((and testp (null key)
+                (member test (list #'eq #'eql #'equal #'equalp)))
+           (multiple-value-bind (len shorter-list)
+               (do ((length 0 (1+ length))
+                    (l1 list1 (cdr l1))
+                    (l2 list2 (cdr l2)))
+                   ((cond ((null l1)
+                           (return (values length list1)))
+                          ((null l2)
+                           (return (values length list2))))))
+             (when (< len 20)
+               (return-from set-difference (default-impl list1 list2)))
+             (flet ((build-hash (len list)
+                      (let ((hashtable (make-hash-table :test test :size len)))
+                        (dolist (item list)
+                          (setf (gethash item hashtable) t))
+                        hashtable)))
+               (cond ((eq shorter-list list2)
+                      (let ((hashtable (build-hash len list2))
+                            diff)
+                        (dolist (item list1)
+                          (unless (gethash item hashtable)
+                            (push item diff)))
+                        diff))
+                     ((eq shorter-list list1)
+                      (let ((hashtable (build-hash len list1)))
+                        (dolist (item list2)
+                          (when (gethash item hashtable)
+                            (remhash item hashtable)))
+                        (loop for item being the hash-keys of hashtable
+                              collect item)))))))
+          (t
+           (default-impl list1 list2)))))
 
 
 (defun nset-difference (list1 list2 &key key
-- 
GitLab


From 5e47bcfc5129d5d7b47b11060adf21a716930838 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Wed, 14 Jun 2023 09:08:48 -0700
Subject: [PATCH 02/22] Refactoring of list to hashtable to its own function

Move the code for converting a list to a hash table to its own
function.  Also support using a key function.

Update set-difference to use this new function.
---
 src/code/list.lisp | 95 +++++++++++++++++++++++++++-------------------
 1 file changed, 57 insertions(+), 38 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 21bd6531e..de2084d70 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -745,6 +745,41 @@
       (cons item list)))
 
 
+;; Convert a list to a hashtable.  Given 2 lists, find the shorter of
+;; the two lists and add the shorter list to a hashtable.  
+(defun list-to-hashtable (list1 list2 &key test test-not key)
+  ;; Don't currently support test-not when converting a list to a hashtable
+  (unless test-not
+    (let ((hash-test (let ((test-fn (if (and (symbolp test)
+					     (fboundp test))
+					(fdefinition test)
+					test)))
+		       (cond ((eql test-fn #'eq) 'eq)
+			     ((eql test-fn #'eql) 'eql)
+			     ((eql test-fn #'equal) 'equal)
+			     ((eql test-fn #'equalp) 'equalp)))))
+      (unless hash-test
+	(return-from list-to-hashtable (values nil nil)))
+      (multiple-value-bind (len shorter-list)
+          (do ((length 0 (1+ length))
+               (l1 list1 (cdr l1))
+               (l2 list2 (cdr l2)))
+              ((cond ((null l1)
+                      (return (values length list1)))
+                     ((null l2)
+                      (return (values length list2))))))
+        (when (< len 15)
+          (return-from list-to-hashtable (values nil nil)))
+        (flet ((build-hash (len list)
+                 (let ((hashtable (make-hash-table :test test :size len)))
+                   (dolist (item list)
+                     (setf (gethash (apply-key key item) hashtable) item))
+                   hashtable)))
+          (cond ((eq shorter-list list2)
+		 (values (build-hash len list2) list2))
+                ((eq shorter-list list1)
+		 (values (build-hash len list1) list1))))))))
+
 ;;; UNION -- Public.
 ;;;
 ;;; This function assumes list2 is the result, adding to it from list1 as
@@ -812,53 +847,37 @@
 	  (setq list1 (Cdr list1))))
     res))
 
-(defun set-difference (list1 list2 &key key
-				     (test #'eql testp) (test-not nil notp))
+(defun set-difference (list1 list2 &key key (test #'eql testp) (test-not nil notp))
   "Returns the elements of list1 which are not in list2."
   (declare (inline member))
   (if (and testp notp)
       (error "Test and test-not both supplied."))
-  (flet ((default-impl (list1 list2)
-           (if (null list2)
+  (multiple-value-bind (hashtable shorter-list)
+      (list-to-hashtable list1 list2 :key key :test test :test-not test-not)
+    (cond ((null hashtable)
+	   ;; Default implementation because we didn't create the hash
+	   ;; table.
+	   (if (null list2)
                list1
                (let ((res nil))
 		 (dolist (elt list1)
                    (if (not (with-set-keys (member (apply-key key elt) list2)))
                        (push elt res)))
-		 res))))
-    (cond ((and testp (null key)
-                (member test (list #'eq #'eql #'equal #'equalp)))
-           (multiple-value-bind (len shorter-list)
-               (do ((length 0 (1+ length))
-                    (l1 list1 (cdr l1))
-                    (l2 list2 (cdr l2)))
-                   ((cond ((null l1)
-                           (return (values length list1)))
-                          ((null l2)
-                           (return (values length list2))))))
-             (when (< len 20)
-               (return-from set-difference (default-impl list1 list2)))
-             (flet ((build-hash (len list)
-                      (let ((hashtable (make-hash-table :test test :size len)))
-                        (dolist (item list)
-                          (setf (gethash item hashtable) t))
-                        hashtable)))
-               (cond ((eq shorter-list list2)
-                      (let ((hashtable (build-hash len list2))
-                            diff)
-                        (dolist (item list1)
-                          (unless (gethash item hashtable)
-                            (push item diff)))
-                        diff))
-                     ((eq shorter-list list1)
-                      (let ((hashtable (build-hash len list1)))
-                        (dolist (item list2)
-                          (when (gethash item hashtable)
-                            (remhash item hashtable)))
-                        (loop for item being the hash-keys of hashtable
-                              collect item)))))))
-          (t
-           (default-impl list1 list2)))))
+		 res)))
+	  ((eq shorter-list list2)
+	   ;; list2 was placed in hash table.
+	   (let (diff)
+             (dolist (item list1)
+	       (unless (gethash (apply-key key item) hashtable)
+                 (push item diff)))
+             diff))
+          ((eq shorter-list list1)
+	   ;; list1 was placed in the hash table.
+           (dolist (item list2)
+	     (when (gethash (apply-key key item) hashtable)
+               (remhash item hashtable)))
+           (loop for item being the hash-values of hashtable
+                 collect item)))))
 
 
 (defun nset-difference (list1 list2 &key key
-- 
GitLab


From 42fd6d23ea3a140aa50cb0a2f7d555b6730fae31 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Thu, 15 Jun 2023 15:23:03 -0700
Subject: [PATCH 03/22] Make hashtable support multiset for list1

It's important that for list1 the hashtable support multisets since
list1 can be.  This is needed so duplicates in list1 show up in the
output.

This variant was from @cshapiro.
---
 src/code/list.lisp | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index de2084d70..b9051a394 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -770,15 +770,16 @@
                       (return (values length list2))))))
         (when (< len 15)
           (return-from list-to-hashtable (values nil nil)))
-        (flet ((build-hash (len list)
-                 (let ((hashtable (make-hash-table :test test :size len)))
-                   (dolist (item list)
-                     (setf (gethash (apply-key key item) hashtable) item))
-                   hashtable)))
-          (cond ((eq shorter-list list2)
-		 (values (build-hash len list2) list2))
-                ((eq shorter-list list1)
-		 (values (build-hash len list1) list1))))))))
+        (cond ((eq shorter-list list2)
+	       (let ((hashtable (make-hash-table :test test :size len)))
+                 (dolist (item list2)
+                   (setf (gethash (apply-key key item) hashtable) item))
+                 (values hashtable list2)))
+              ((eq shorter-list list1)
+	       (let ((hashtable (make-hash-table :test test :size len)))
+		 (dolist (item list1)
+     		   (push item (gethash (apply-key key item) hashtable)))
+		 (values hashtable list1))))))))
 
 ;;; UNION -- Public.
 ;;;
@@ -874,10 +875,13 @@
           ((eq shorter-list list1)
 	   ;; list1 was placed in the hash table.
            (dolist (item list2)
-	     (when (gethash (apply-key key item) hashtable)
+	     (unless (eq hashtable (gethash (apply-key key item) hashtable hashtable))
                (remhash item hashtable)))
-           (loop for item being the hash-values of hashtable
-                 collect item)))))
+	   (let ((result '()))
+	     (maphash #'(lambda (key value)
+			  (declare (ignore key))
+			  (setq result (nconc result value)))
+		      hashtable))))))
 
 
 (defun nset-difference (list1 list2 &key key
-- 
GitLab


From 85c8c1dfb7aa0c0b2fc4d15a23bb9204fd7025bc Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Thu, 15 Jun 2023 15:26:43 -0700
Subject: [PATCH 04/22] Reverse the output

Reverse the output for the case when list1 is longer so that the order
of the items matches the order of the input.
---
 src/code/list.lisp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index b9051a394..4a319c467 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -871,7 +871,7 @@
              (dolist (item list1)
 	       (unless (gethash (apply-key key item) hashtable)
                  (push item diff)))
-             diff))
+             (nreverse diff)))
           ((eq shorter-list list1)
 	   ;; list1 was placed in the hash table.
            (dolist (item list2)
-- 
GitLab


From f9485d0873657a8e83336341e4eb6bec5a6c4807 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Mon, 19 Jun 2023 12:46:39 -0700
Subject: [PATCH 05/22] Handle the case of NIL in list2.

When list2 is in the hash table and it contains NIL and list1 also
contains NIL, we were erroneously including NIL in the difference.
This was because we weren't checking the second return value for
gethash.

Also, for consistency, don't reverse the set of differences.  This
makes output from the hash-table version match the list version.

Finally, add a defparameter to specify the length threshold for
converting a list to hashtable.  This is mostly for testing to
determine an appropriate value.
---
 src/code/list.lisp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 4a319c467..c88c4f909 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -744,6 +744,8 @@
       list
       (cons item list)))
 
+(defparameter *min-list-length-for-hashtable*
+  15)
 
 ;; Convert a list to a hashtable.  Given 2 lists, find the shorter of
 ;; the two lists and add the shorter list to a hashtable.  
@@ -768,7 +770,7 @@
                       (return (values length list1)))
                      ((null l2)
                       (return (values length list2))))))
-        (when (< len 15)
+        (when (< len *min-list-length-for-hashtable*)
           (return-from list-to-hashtable (values nil nil)))
         (cond ((eq shorter-list list2)
 	       (let ((hashtable (make-hash-table :test test :size len)))
@@ -869,9 +871,9 @@
 	   ;; list2 was placed in hash table.
 	   (let (diff)
              (dolist (item list1)
-	       (unless (gethash (apply-key key item) hashtable)
+	       (unless (nth-value 1 (gethash (apply-key key item) hashtable))
                  (push item diff)))
-             (nreverse diff)))
+             diff))
           ((eq shorter-list list1)
 	   ;; list1 was placed in the hash table.
            (dolist (item list2)
-- 
GitLab


From 653dc42ff6285a475709fe25fa9f5d54c998f4ab Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Mon, 19 Jun 2023 13:19:51 -0700
Subject: [PATCH 06/22] Forgot to return the value

When list1 was in the hashtable, we forgot to return values from the
hashtable after computing the return value.
---
 src/code/list.lisp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index c88c4f909..039f7d201 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -883,7 +883,8 @@
 	     (maphash #'(lambda (key value)
 			  (declare (ignore key))
 			  (setq result (nconc result value)))
-		      hashtable))))))
+		      hashtable)
+	     result)))))
 
 
 (defun nset-difference (list1 list2 &key key
-- 
GitLab


From ac30347e46ecac4d43d5646c628e8a4a0eeb9065 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Mon, 19 Jun 2023 13:45:44 -0700
Subject: [PATCH 07/22] Prefer list2 when both lists have the same length

Also add some comments.
---
 src/code/list.lisp | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 039f7d201..7a52f8ef2 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -744,11 +744,13 @@
       list
       (cons item list)))
 
+;; The minimum length of a list before we can use a hashtable
 (defparameter *min-list-length-for-hashtable*
   15)
 
 ;; Convert a list to a hashtable.  Given 2 lists, find the shorter of
-;; the two lists and add the shorter list to a hashtable.  
+;; the two lists and add the shorter list to a hashtable.  Returns the
+;; hashtable and the shorter list.
 (defun list-to-hashtable (list1 list2 &key test test-not key)
   ;; Don't currently support test-not when converting a list to a hashtable
   (unless test-not
@@ -763,13 +765,18 @@
       (unless hash-test
 	(return-from list-to-hashtable (values nil nil)))
       (multiple-value-bind (len shorter-list)
+	  ;; Find the list with the shorter length.  If they're they
+	  ;; same, we prefer the second list to the first list since
+	  ;; the hashtable implementation is slightly simplier.
           (do ((length 0 (1+ length))
                (l1 list1 (cdr l1))
                (l2 list2 (cdr l2)))
-              ((cond ((null l1)
-                      (return (values length list1)))
-                     ((null l2)
-                      (return (values length list2))))))
+              ((cond ((null l2)
+                      (return (values length list2)))
+		     ((null l1)
+                      (return (values length list1))))))
+	;; If the list is too short, the hashtable makes things
+	;; slower.  We also need to balance memory usage.
         (when (< len *min-list-length-for-hashtable*)
           (return-from list-to-hashtable (values nil nil)))
         (cond ((eq shorter-list list2)
-- 
GitLab


From 57f0451ca3e5a2af12b06b37138b2465dc65960c Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Mon, 19 Jun 2023 13:48:29 -0700
Subject: [PATCH 08/22] Allow quick exit in set-difference

Also, now that set-difference is pretty large, don't declare it as
maybe-inline.
---
 src/code/list.lisp | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 7a52f8ef2..e7df88fa2 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -45,7 +45,7 @@
 	  tree-equal list-length nth %setnth nthcdr last make-list append
 	  copy-list copy-alist copy-tree revappend nconc nreconc butlast
 	  nbutlast ldiff member member-if member-if-not tailp adjoin union
-	  nunion intersection nintersection set-difference nset-difference
+	  nunion intersection nintersection nset-difference
 	  set-exclusive-or nset-exclusive-or subsetp acons pairlis assoc
 	  assoc-if assoc-if-not rassoc rassoc-if rassoc-if-not subst subst-if
 	  subst-if-not nsubst nsubst-if nsubst-if-not sublis nsublis))
@@ -862,18 +862,20 @@
   (declare (inline member))
   (if (and testp notp)
       (error "Test and test-not both supplied."))
+  ;; Quick exit
+  (when (null list)
+    (return-from set-difference list1))
+
   (multiple-value-bind (hashtable shorter-list)
       (list-to-hashtable list1 list2 :key key :test test :test-not test-not)
     (cond ((null hashtable)
 	   ;; Default implementation because we didn't create the hash
 	   ;; table.
-	   (if (null list2)
-               list1
-               (let ((res nil))
-		 (dolist (elt list1)
-                   (if (not (with-set-keys (member (apply-key key elt) list2)))
-                       (push elt res)))
-		 res)))
+           (let ((res nil))
+	     (dolist (elt list1)
+               (if (not (with-set-keys (member (apply-key key elt) list2)))
+                   (push elt res)))
+	     res))
 	  ((eq shorter-list list2)
 	   ;; list2 was placed in hash table.
 	   (let (diff)
-- 
GitLab


From bc7826a734123f627eccc4ff29d95b01c7043f78 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Mon, 19 Jun 2023 14:26:19 -0700
Subject: [PATCH 09/22] Fix typo in set-difference

The early exist for when list2 is empty had a typo with `list` instead
of `list2`.
---
 src/code/list.lisp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index e7df88fa2..1047bad90 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -863,7 +863,7 @@
   (if (and testp notp)
       (error "Test and test-not both supplied."))
   ;; Quick exit
-  (when (null list)
+  (when (null list2)
     (return-from set-difference list1))
 
   (multiple-value-bind (hashtable shorter-list)
-- 
GitLab


From 28aafe047bc4bf7ee9485c979b423e92a8dc304d Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Mon, 19 Jun 2023 18:12:15 -0700
Subject: [PATCH 10/22] Add some simple set-difference tests with hashtables.

---
 tests/sets.lisp | 74 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 tests/sets.lisp

diff --git a/tests/sets.lisp b/tests/sets.lisp
new file mode 100644
index 000000000..bfd97035c
--- /dev/null
+++ b/tests/sets.lisp
@@ -0,0 +1,74 @@
+;; Tests for sets
+
+(defpackage :sets-tests
+  (:use :cl :lisp-unit))
+
+(in-package "SETS-TESTS")
+
+(define-test set-diff.hash-eql
+    (:tag :issues)
+  ;; For set-difference to use hashtables by making the threshold
+  ;; small.
+  (let ((lisp::*min-list-length-for-hashtable* 2))
+    (assert-equal '(2 2 1)
+		  (set-difference '(1 2 2 3) '(3 4)))
+    (assert-equal '(1 2 2)
+		  (set-difference '(1 2 2 3) '(3 4 5 6 7 8)))
+    (assert-equal '(2 2 1)
+		  (set-difference '(1 2 2 3) '(3 4)
+				  :test #'eql))
+    (assert-equal '(1 2 2)
+		  (set-difference '(1 2 2 3) '(3 4 5 6 7 8)
+				  :test #'eql))))
+
+(define-test set-diff.hash-eq
+    (:tag :issues)
+  (let ((lisp::*min-list-length-for-hashtable* 2))
+    (assert-equal '(b b a)
+		  (set-difference '(a b b c) '(c d e) :test 'eq))
+    (assert-equal '(a b b)
+		  (set-difference '(a b b c) '(c d e f g h) :test 'eq))
+    (assert-equal '(b b a)
+		  (set-difference '(a b b c) '(c d e) :test #'eq))
+    (assert-equal '(a b b)
+		  (set-difference '(a b b c) '(c d e f g h) :test #'eq))))
+
+(define-test set-diff.hash-equal
+    (:tag :issues)
+  (let ((lisp::*min-list-length-for-hashtable* 2))
+    (assert-equal '("b" "b" "a")
+		  (set-difference '("a" "b" "b" "c")
+				  '("c" "d" "e")
+				  :test 'equal))
+    (assert-equal '("a" "b" "b")
+		  (set-difference '("a" "b" "b" "c")
+				  '("c" "d" "e" "f" "g" "h")
+				  :test 'equal))
+    (assert-equal '("b" "b" "a")
+		  (set-difference '("a" "b" "b" "c")
+				  '("c" "d" "e")
+				  :test #'equal))
+    (assert-equal '("a" "b" "b")
+		  (set-difference '("a" "b" "b" "c")
+				  '("c" "d" "e" "f" "g" "h")
+				  :test #'equal))))
+
+(define-test set-diff.hash-equalp
+    (:tag :issues)
+  (let ((lisp::*min-list-length-for-hashtable* 2))
+    (assert-equal '("b" "b" "a")
+		  (set-difference '("a" "b" "b" "c")
+				  '("C" "d" "e")
+				  :test 'equalp))
+    (assert-equal '("a" "b" "b")
+		  (set-difference '("a" "b" "b" "C")
+				  '("c" "D" "e" "f" "g" "h")
+				  :test 'equalp))
+    (assert-equal '("b" "b" "a")
+		  (set-difference '("a" "b" "b" "c")
+				  '("C" "d" "e")
+				  :test #'equalp))
+    (assert-equal '("a" "b" "b")
+		  (set-difference '("a" "b" "b" "C")
+				  '("c" "D" "e" "f" "g" "h")
+				  :test #'equalp))))
-- 
GitLab


From 7e598eb28797a2310bab7dd29dc2cdaabba4b386 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Thu, 22 Jun 2023 16:07:33 -0700
Subject: [PATCH 11/22] Break up functions into smaller pieces

Split out the key parts of list-to-hasthable and set-difference to
make it easier to profile the cost of doing the individual items, like
making the hashtables from list1 and list2 and computing the set
difference using the two different hashtables.
---
 src/code/list.lisp | 51 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 45 insertions(+), 6 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 1047bad90..9b2b26c2d 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -748,6 +748,18 @@
 (defparameter *min-list-length-for-hashtable*
   15)
 
+(defun init-hashtable-list1 (list1 len &key key test)
+  (let ((hashtable (make-hash-table :test test :size len)))
+    (dolist (item list1)
+      (push item (gethash (apply-key key item) hashtable)))
+    (values hashtable list1)))
+
+(defun init-hashtable-list2 (list2 len &key key test)
+  (let ((hashtable (make-hash-table :test test :size len)))
+    (dolist (item list2)
+      (setf (gethash (apply-key key item) hashtable) item))
+    (values hashtable list2)))
+
 ;; Convert a list to a hashtable.  Given 2 lists, find the shorter of
 ;; the two lists and add the shorter list to a hashtable.  Returns the
 ;; hashtable and the shorter list.
@@ -771,24 +783,28 @@
           (do ((length 0 (1+ length))
                (l1 list1 (cdr l1))
                (l2 list2 (cdr l2)))
-              ((cond ((null l2)
+              ((cond ((endp l2)
                       (return (values length list2)))
-		     ((null l1)
+		     ((endp l1)
                       (return (values length list1))))))
 	;; If the list is too short, the hashtable makes things
 	;; slower.  We also need to balance memory usage.
         (when (< len *min-list-length-for-hashtable*)
           (return-from list-to-hashtable (values nil nil)))
         (cond ((eq shorter-list list2)
+	       #+nil
 	       (let ((hashtable (make-hash-table :test test :size len)))
                  (dolist (item list2)
                    (setf (gethash (apply-key key item) hashtable) item))
-                 (values hashtable list2)))
+                 (values hashtable list2))
+	       (init-hashtable-list2 list2 len :key key :test test))
               ((eq shorter-list list1)
+	       #+nil
 	       (let ((hashtable (make-hash-table :test test :size len)))
 		 (dolist (item list1)
      		   (push item (gethash (apply-key key item) hashtable)))
-		 (values hashtable list1))))))))
+		 (values hashtable list1))
+	       (init-hashtable-list1 list1 len :key key :test test)))))))
 
 ;;; UNION -- Public.
 ;;;
@@ -857,6 +873,24 @@
 	  (setq list1 (Cdr list1))))
     res))
 
+(defun set-diff-hash2 (list1 hashtable &key key)
+  (let (diff)
+    (dolist (item list1)
+      (unless (nth-value 1 (gethash (apply-key key item) hashtable))
+        (push item diff)))
+    diff))
+
+(defun set-diff-hash1 (list2 hashtable &key key)
+  (dolist (item list2)
+    (unless (eq hashtable (gethash (apply-key key item) hashtable hashtable))
+      (remhash item hashtable)))
+  (let ((result '()))
+    (maphash #'(lambda (key value)
+		 (declare (ignore key))
+		 (setq result (nconc result value)))
+	     hashtable)
+    result))  
+
 (defun set-difference (list1 list2 &key key (test #'eql testp) (test-not nil notp))
   "Returns the elements of list1 which are not in list2."
   (declare (inline member))
@@ -878,22 +912,27 @@
 	     res))
 	  ((eq shorter-list list2)
 	   ;; list2 was placed in hash table.
+	   #+nil
 	   (let (diff)
              (dolist (item list1)
 	       (unless (nth-value 1 (gethash (apply-key key item) hashtable))
                  (push item diff)))
-             diff))
+             diff)
+	   (set-diff-hash2 list1 hashtable :key key))
           ((eq shorter-list list1)
 	   ;; list1 was placed in the hash table.
+	   #+nil
            (dolist (item list2)
 	     (unless (eq hashtable (gethash (apply-key key item) hashtable hashtable))
                (remhash item hashtable)))
+	   #+nil
 	   (let ((result '()))
 	     (maphash #'(lambda (key value)
 			  (declare (ignore key))
 			  (setq result (nconc result value)))
 		      hashtable)
-	     result)))))
+	     result)
+	   (set-diff-hash1 list2 hashtable :key key)))))
 
 
 (defun nset-difference (list1 list2 &key key
-- 
GitLab


From 2e46d4581f00c9d3a1df8b3fddeec396a48453d4 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Sat, 24 Jun 2023 08:35:33 -0700
Subject: [PATCH 12/22] Modify code so we only hash list2

Tests for when we hash list1 containing fixnums show that we're always
much slower that not hashing.  But when list1 consists of strings,
we're always better hashing list1.  So complicated.

So for now, just hash list2, like what clisp does.
---
 src/code/list.lisp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 9b2b26c2d..3b901a1e8 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -785,6 +785,7 @@
                (l2 list2 (cdr l2)))
               ((cond ((endp l2)
                       (return (values length list2)))
+		     #+nil
 		     ((endp l1)
                       (return (values length list1))))))
 	;; If the list is too short, the hashtable makes things
-- 
GitLab


From f44c4b811c7f33f9a12eab8afc8f40709c5f4e84 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Mon, 24 Jul 2023 14:51:30 -0700
Subject: [PATCH 13/22] Clean up set-diff code

Remove unused functions having to do with hashing the first list.
Modify list-to-hashtable to only want one list to be hashed and only
returning one value (the hashtable).  Manually inline set-diff-hash2.

Apply Carl's suggest to use ecase instead of cond.

Update tests because the hash table returns the results in the same
order as the list version.
---
 src/code/list.lisp | 125 ++++++++++-----------------------------------
 1 file changed, 28 insertions(+), 97 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 3b901a1e8..cf6d4179f 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -748,64 +748,30 @@
 (defparameter *min-list-length-for-hashtable*
   15)
 
-(defun init-hashtable-list1 (list1 len &key key test)
-  (let ((hashtable (make-hash-table :test test :size len)))
-    (dolist (item list1)
-      (push item (gethash (apply-key key item) hashtable)))
-    (values hashtable list1)))
-
-(defun init-hashtable-list2 (list2 len &key key test)
-  (let ((hashtable (make-hash-table :test test :size len)))
-    (dolist (item list2)
-      (setf (gethash (apply-key key item) hashtable) item))
-    (values hashtable list2)))
-
-;; Convert a list to a hashtable.  Given 2 lists, find the shorter of
-;; the two lists and add the shorter list to a hashtable.  Returns the
-;; hashtable and the shorter list.
-(defun list-to-hashtable (list1 list2 &key test test-not key)
+;; Convert a list to a hashtable.  The hashtable does not handle
+;; duplicated values in the list.  Returns the hashtable.
+(defun list-to-hashtable (list &key test test-not key)
   ;; Don't currently support test-not when converting a list to a hashtable
   (unless test-not
     (let ((hash-test (let ((test-fn (if (and (symbolp test)
 					     (fboundp test))
 					(fdefinition test)
 					test)))
-		       (cond ((eql test-fn #'eq) 'eq)
-			     ((eql test-fn #'eql) 'eql)
-			     ((eql test-fn #'equal) 'equal)
-			     ((eql test-fn #'equalp) 'equalp)))))
+		       (case test-fn
+			 (#'eq 'eq)
+			 (#'eql 'eql)
+			 (#'equal 'equal)
+			 (#'equalp 'equalp)))))
       (unless hash-test
-	(return-from list-to-hashtable (values nil nil)))
-      (multiple-value-bind (len shorter-list)
-	  ;; Find the list with the shorter length.  If they're they
-	  ;; same, we prefer the second list to the first list since
-	  ;; the hashtable implementation is slightly simplier.
-          (do ((length 0 (1+ length))
-               (l1 list1 (cdr l1))
-               (l2 list2 (cdr l2)))
-              ((cond ((endp l2)
-                      (return (values length list2)))
-		     #+nil
-		     ((endp l1)
-                      (return (values length list1))))))
-	;; If the list is too short, the hashtable makes things
-	;; slower.  We also need to balance memory usage.
-        (when (< len *min-list-length-for-hashtable*)
-          (return-from list-to-hashtable (values nil nil)))
-        (cond ((eq shorter-list list2)
-	       #+nil
-	       (let ((hashtable (make-hash-table :test test :size len)))
-                 (dolist (item list2)
-                   (setf (gethash (apply-key key item) hashtable) item))
-                 (values hashtable list2))
-	       (init-hashtable-list2 list2 len :key key :test test))
-              ((eq shorter-list list1)
-	       #+nil
-	       (let ((hashtable (make-hash-table :test test :size len)))
-		 (dolist (item list1)
-     		   (push item (gethash (apply-key key item) hashtable)))
-		 (values hashtable list1))
-	       (init-hashtable-list1 list1 len :key key :test test)))))))
+	(return-from list-to-hashtable nil))
+      ;; If the list is too short, the hashtable makes things
+      ;; slower.  We also need to balance memory usage.
+      (when (< (length list) *min-list-length-for-hashtable*)
+        (return-from list-to-hashtable nil))
+      (let ((hashtable (make-hash-table :test test :size len)))
+	(dolist (item list)
+	  (setf (gethash (apply-key key item) hashtable) item))
+	hashtable))))
 
 ;;; UNION -- Public.
 ;;;
@@ -874,24 +840,6 @@
 	  (setq list1 (Cdr list1))))
     res))
 
-(defun set-diff-hash2 (list1 hashtable &key key)
-  (let (diff)
-    (dolist (item list1)
-      (unless (nth-value 1 (gethash (apply-key key item) hashtable))
-        (push item diff)))
-    diff))
-
-(defun set-diff-hash1 (list2 hashtable &key key)
-  (dolist (item list2)
-    (unless (eq hashtable (gethash (apply-key key item) hashtable hashtable))
-      (remhash item hashtable)))
-  (let ((result '()))
-    (maphash #'(lambda (key value)
-		 (declare (ignore key))
-		 (setq result (nconc result value)))
-	     hashtable)
-    result))  
-
 (defun set-difference (list1 list2 &key key (test #'eql testp) (test-not nil notp))
   "Returns the elements of list1 which are not in list2."
   (declare (inline member))
@@ -901,40 +849,23 @@
   (when (null list2)
     (return-from set-difference list1))
 
-  (multiple-value-bind (hashtable shorter-list)
-      (list-to-hashtable list1 list2 :key key :test test :test-not test-not)
-    (cond ((null hashtable)
+  (let ((hashtable 
+	  (list-to-hashtable list2 :key key :test test :test-not test-not)))
+    (cond (hashtable
+	   ;; list2 was placed in hash table.
+	   (let (diff)
+	     (dolist (item list1)
+	       (unless (nth-value 1 (gethash (apply-key key item) hashtable))
+		 (push item diff)))
+	     diff))
+	  ((null hashtable)
 	   ;; Default implementation because we didn't create the hash
 	   ;; table.
            (let ((res nil))
 	     (dolist (elt list1)
                (if (not (with-set-keys (member (apply-key key elt) list2)))
                    (push elt res)))
-	     res))
-	  ((eq shorter-list list2)
-	   ;; list2 was placed in hash table.
-	   #+nil
-	   (let (diff)
-             (dolist (item list1)
-	       (unless (nth-value 1 (gethash (apply-key key item) hashtable))
-                 (push item diff)))
-             diff)
-	   (set-diff-hash2 list1 hashtable :key key))
-          ((eq shorter-list list1)
-	   ;; list1 was placed in the hash table.
-	   #+nil
-           (dolist (item list2)
-	     (unless (eq hashtable (gethash (apply-key key item) hashtable hashtable))
-               (remhash item hashtable)))
-	   #+nil
-	   (let ((result '()))
-	     (maphash #'(lambda (key value)
-			  (declare (ignore key))
-			  (setq result (nconc result value)))
-		      hashtable)
-	     result)
-	   (set-diff-hash1 list2 hashtable :key key)))))
-
+	     res)))))
 
 (defun nset-difference (list1 list2 &key key
 			      (test #'eql testp) (test-not nil notp))
-- 
GitLab


From 12f6ff0e193bbe277e9d4277612b5afa99a8c42d Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Mon, 24 Jul 2023 15:35:18 -0700
Subject: [PATCH 14/22] Update set-diff tests

The hash table returns the results in the same order as the list
version.
---
 tests/sets.lisp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/sets.lisp b/tests/sets.lisp
index bfd97035c..0b5275e31 100644
--- a/tests/sets.lisp
+++ b/tests/sets.lisp
@@ -12,12 +12,12 @@
   (let ((lisp::*min-list-length-for-hashtable* 2))
     (assert-equal '(2 2 1)
 		  (set-difference '(1 2 2 3) '(3 4)))
-    (assert-equal '(1 2 2)
+    (assert-equal '(2 2 1)
 		  (set-difference '(1 2 2 3) '(3 4 5 6 7 8)))
     (assert-equal '(2 2 1)
 		  (set-difference '(1 2 2 3) '(3 4)
 				  :test #'eql))
-    (assert-equal '(1 2 2)
+    (assert-equal '(2 2 1)
 		  (set-difference '(1 2 2 3) '(3 4 5 6 7 8)
 				  :test #'eql))))
 
@@ -26,11 +26,11 @@
   (let ((lisp::*min-list-length-for-hashtable* 2))
     (assert-equal '(b b a)
 		  (set-difference '(a b b c) '(c d e) :test 'eq))
-    (assert-equal '(a b b)
+    (assert-equal '(b b a)
 		  (set-difference '(a b b c) '(c d e f g h) :test 'eq))
     (assert-equal '(b b a)
 		  (set-difference '(a b b c) '(c d e) :test #'eq))
-    (assert-equal '(a b b)
+    (assert-equal '(b b a)
 		  (set-difference '(a b b c) '(c d e f g h) :test #'eq))))
 
 (define-test set-diff.hash-equal
@@ -40,7 +40,7 @@
 		  (set-difference '("a" "b" "b" "c")
 				  '("c" "d" "e")
 				  :test 'equal))
-    (assert-equal '("a" "b" "b")
+    (assert-equal '("b" "b" "a")
 		  (set-difference '("a" "b" "b" "c")
 				  '("c" "d" "e" "f" "g" "h")
 				  :test 'equal))
@@ -48,7 +48,7 @@
 		  (set-difference '("a" "b" "b" "c")
 				  '("c" "d" "e")
 				  :test #'equal))
-    (assert-equal '("a" "b" "b")
+    (assert-equal '("b" "b" "a")
 		  (set-difference '("a" "b" "b" "c")
 				  '("c" "d" "e" "f" "g" "h")
 				  :test #'equal))))
@@ -60,7 +60,7 @@
 		  (set-difference '("a" "b" "b" "c")
 				  '("C" "d" "e")
 				  :test 'equalp))
-    (assert-equal '("a" "b" "b")
+    (assert-equal '("b" "b" "a")
 		  (set-difference '("a" "b" "b" "C")
 				  '("c" "D" "e" "f" "g" "h")
 				  :test 'equalp))
@@ -68,7 +68,7 @@
 		  (set-difference '("a" "b" "b" "c")
 				  '("C" "d" "e")
 				  :test #'equalp))
-    (assert-equal '("a" "b" "b")
+    (assert-equal '("b" "b" "a")
 		  (set-difference '("a" "b" "b" "C")
 				  '("c" "D" "e" "f" "g" "h")
 				  :test #'equalp))))
-- 
GitLab


From 42e2d9be8f524234b178c1ca97c862031fc740cb Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Wed, 26 Jul 2023 07:26:06 -0700
Subject: [PATCH 15/22] Apply suggestions from review, cleaning up some
 oddities.

---
 src/code/list.lisp | 43 ++++++++++++++++++++-----------------------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index cf6d4179f..1074cbd20 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -750,28 +750,25 @@
 
 ;; Convert a list to a hashtable.  The hashtable does not handle
 ;; duplicated values in the list.  Returns the hashtable.
-(defun list-to-hashtable (list &key test test-not key)
+(defun list-to-hashtable (list test test-not key)
   ;; Don't currently support test-not when converting a list to a hashtable
   (unless test-not
-    (let ((hash-test (let ((test-fn (if (and (symbolp test)
-					     (fboundp test))
-					(fdefinition test)
-					test)))
-		       (case test-fn
-			 (#'eq 'eq)
-			 (#'eql 'eql)
-			 (#'equal 'equal)
-			 (#'equalp 'equalp)))))
+    (let ((hash-test (case test
+			 ((#'eq 'eq) 'eq)
+			 ((#'eql 'eq) 'eql)
+			 ((#'equal 'equal) 'equal)
+			 ((#'equalp 'equalp) 'equalp))))
       (unless hash-test
 	(return-from list-to-hashtable nil))
       ;; If the list is too short, the hashtable makes things
       ;; slower.  We also need to balance memory usage.
-      (when (< (length list) *min-list-length-for-hashtable*)
-        (return-from list-to-hashtable nil))
-      (let ((hashtable (make-hash-table :test test :size len)))
-	(dolist (item list)
-	  (setf (gethash (apply-key key item) hashtable) item))
-	hashtable))))
+      (let ((len (length list)))
+	(when (< len *min-list-length-for-hashtable*)
+          (return-from list-to-hashtable nil))
+	(let ((hashtable (make-hash-table :test hash-test :size len)))
+	  (dolist (item list)
+	    (setf (gethash (apply-key key item) hashtable) item))
+	  hashtable)))))
 
 ;;; UNION -- Public.
 ;;;
@@ -850,21 +847,21 @@
     (return-from set-difference list1))
 
   (let ((hashtable 
-	  (list-to-hashtable list2 :key key :test test :test-not test-not)))
+	  (list-to-hashtable list2 key test test-not)))
     (cond (hashtable
 	   ;; list2 was placed in hash table.
-	   (let (diff)
+	   (let ((res nil))
 	     (dolist (item list1)
 	       (unless (nth-value 1 (gethash (apply-key key item) hashtable))
-		 (push item diff)))
-	     diff))
+		 (push item res)))
+	     res))
 	  ((null hashtable)
 	   ;; Default implementation because we didn't create the hash
 	   ;; table.
            (let ((res nil))
-	     (dolist (elt list1)
-               (if (not (with-set-keys (member (apply-key key elt) list2)))
-                   (push elt res)))
+	     (dolist (item list1)
+	       (if (not (with-set-keys (member (apply-key key item) list2)))
+                   (push item res)))
 	     res)))))
 
 (defun nset-difference (list1 list2 &key key
-- 
GitLab


From c09004a15331d191b5dcfe717fad39972e1ce2e4 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Wed, 26 Jul 2023 14:34:09 -0700
Subject: [PATCH 16/22] Make min-list-length-for-hashtable a defconstant

Update name to use `+foo+` convention for constants.
---
 src/code/list.lisp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 1074cbd20..dcad51d0d 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -744,8 +744,9 @@
       list
       (cons item list)))
 
-;; The minimum length of a list before we can use a hashtable
-(defparameter *min-list-length-for-hashtable*
+;; The minimum length of a list before we can use a hashtable.  This
+;; was determined experimentally.
+(defconstant +min-list-length-for-hashtable+
   15)
 
 ;; Convert a list to a hashtable.  The hashtable does not handle
@@ -763,7 +764,7 @@
       ;; If the list is too short, the hashtable makes things
       ;; slower.  We also need to balance memory usage.
       (let ((len (length list)))
-	(when (< len *min-list-length-for-hashtable*)
+	(when (< len +min-list-length-for-hashtable+)
           (return-from list-to-hashtable nil))
 	(let ((hashtable (make-hash-table :test hash-test :size len)))
 	  (dolist (item list)
-- 
GitLab


From d6aae1fe56163ffb2e2c49c1165146ab6bb52b44 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Wed, 26 Jul 2023 18:35:56 -0700
Subject: [PATCH 17/22] Revert change that converted cond to case

The use of `case` insted of `cond` didn't work as we naively
expected.  Change it back to using `cond`.
---
 src/code/list.lisp | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index dcad51d0d..eaea86d1d 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -754,11 +754,14 @@
 (defun list-to-hashtable (list test test-not key)
   ;; Don't currently support test-not when converting a list to a hashtable
   (unless test-not
-    (let ((hash-test (case test
-			 ((#'eq 'eq) 'eq)
-			 ((#'eql 'eq) 'eql)
-			 ((#'equal 'equal) 'equal)
-			 ((#'equalp 'equalp) 'equalp))))
+    (let ((hash-test (let ((test-fn (if (and (symbolp test)
+                                             (fboundp test))
+                                        (fdefinition test)
+                                        test)))
+                       (cond ((eql test-fn #'eq) 'eq)
+                             ((eql test-fn #'eql) 'eql)
+                             ((eql test-fn #'equal) 'equal)
+                             ((eql test-fn #'equalp) 'equalp)))))
       (unless hash-test
 	(return-from list-to-hashtable nil))
       ;; If the list is too short, the hashtable makes things
-- 
GitLab


From 3fbae4d018b9bc6891effd46c1e0ec71c3538d7f Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Wed, 26 Jul 2023 18:40:57 -0700
Subject: [PATCH 18/22] Fix args to list-to-hashtable

Originally, we had the args in the order `test`, `test-not`, and
`key`.  We really should have `key`, `test`, and `test-not`.  That's
how set-difference was calling `list-to-hashtable`, and it make more
sense for the args to be in this order.
---
 src/code/list.lisp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index eaea86d1d..280a5cad4 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -751,7 +751,7 @@
 
 ;; Convert a list to a hashtable.  The hashtable does not handle
 ;; duplicated values in the list.  Returns the hashtable.
-(defun list-to-hashtable (list test test-not key)
+(defun list-to-hashtable (list key test test-not)
   ;; Don't currently support test-not when converting a list to a hashtable
   (unless test-not
     (let ((hash-test (let ((test-fn (if (and (symbolp test)
-- 
GitLab


From 96e660716071d2727d4264610f52f13acebb3d70 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Wed, 26 Jul 2023 18:59:10 -0700
Subject: [PATCH 19/22] Make +min-list-length-for-hashtable+ a defparameter
 again

Rename it using the `*foo*` convention too.  It's super-convenient to
be able to set this for testing, which we do in the existing test
suite for this.

It's also convenient to be able to do this when doing timing tests so
we can compare what happens when the list version is used vs when the
hashtable is used.
---
 src/code/list.lisp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 280a5cad4..39622c65d 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -746,7 +746,7 @@
 
 ;; The minimum length of a list before we can use a hashtable.  This
 ;; was determined experimentally.
-(defconstant +min-list-length-for-hashtable+
+(defparameter *min-list-length-for-hashtable*
   15)
 
 ;; Convert a list to a hashtable.  The hashtable does not handle
@@ -767,7 +767,7 @@
       ;; If the list is too short, the hashtable makes things
       ;; slower.  We also need to balance memory usage.
       (let ((len (length list)))
-	(when (< len +min-list-length-for-hashtable+)
+	(when (< len *min-list-length-for-hashtable*)
           (return-from list-to-hashtable nil))
 	(let ((hashtable (make-hash-table :test hash-test :size len)))
 	  (dolist (item list)
@@ -855,7 +855,7 @@
     (cond (hashtable
 	   ;; list2 was placed in hash table.
 	   (let ((res nil))
-	     (dolist (item list1)
+	     (doli st (item list1)
 	       (unless (nth-value 1 (gethash (apply-key key item) hashtable))
 		 (push item res)))
 	     res))
-- 
GitLab


From 7ab5d2435fc0e4b32e8603b07b9e0e4f48fbdcba Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Wed, 26 Jul 2023 19:00:59 -0700
Subject: [PATCH 20/22] Add some additional tests for set-difference

Add a test where a key is given to make sure that hashtable works
correctly with a key

Add a test where both a test and a test-not is given.  This must
signal an error.
---
 tests/sets.lisp | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/sets.lisp b/tests/sets.lisp
index 0b5275e31..1a7328981 100644
--- a/tests/sets.lisp
+++ b/tests/sets.lisp
@@ -72,3 +72,20 @@
 		  (set-difference '("a" "b" "b" "C")
 				  '("c" "D" "e" "f" "g" "h")
 				  :test #'equalp))))
+
+;; Simple test that we handle a key correctly
+(define-test set-diff.hash-eql-with-key
+  (let ((lisp::*min-list-length-for-hashtable* 2))
+    (assert-equal '((3 "b") (2 "b"))
+		  (set-difference '((1 "a") (2 "b") (3 "b"))
+				  '((1 "a") (4 "c") (5 "d"))
+				  :key #'first))))
+
+(define-test set-diff.test-and-test-not
+  (assert-error 'simple-error
+		(set-difference '(1 2)
+				'(3 4)
+				:test 'eql
+				:test-not 'eql)))
+
+   
-- 
GitLab


From c3aa0678c04d5197ed783763bd60e61210560774 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Thu, 27 Jul 2023 07:31:17 -0700
Subject: [PATCH 21/22] Fix typo.

---
 src/code/list.lisp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index 39622c65d..c54e7b160 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -855,7 +855,7 @@
     (cond (hashtable
 	   ;; list2 was placed in hash table.
 	   (let ((res nil))
-	     (doli st (item list1)
+	     (dolist (item list1)
 	       (unless (nth-value 1 (gethash (apply-key key item) hashtable))
 		 (push item res)))
 	     res))
-- 
GitLab


From b100ecd403f9ce701532d385460baecdd3088a22 Mon Sep 17 00:00:00 2001
From: Raymond Toy <toy.raymond@gmail.com>
Date: Thu, 27 Jul 2023 17:59:17 -0700
Subject: [PATCH 22/22] Compute list length manual instead of calling length

For some strange reason, ansi-test `set-difference.error.11` fails in
`lisp::list-to-hashtable` which uses `length` to compute the length of
a list.  Let's try computing the length manually ourselves.
---
 src/code/list.lisp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/code/list.lisp b/src/code/list.lisp
index c54e7b160..186ef678b 100644
--- a/src/code/list.lisp
+++ b/src/code/list.lisp
@@ -766,7 +766,11 @@
 	(return-from list-to-hashtable nil))
       ;; If the list is too short, the hashtable makes things
       ;; slower.  We also need to balance memory usage.
-      (let ((len (length list)))
+      (let ((len 0))
+	;; Compute list length ourselves.
+	(dolist (item list)
+	  (declare (ignore item))
+	  (incf len))
 	(when (< len *min-list-length-for-hashtable*)
           (return-from list-to-hashtable nil))
 	(let ((hashtable (make-hash-table :test hash-test :size len)))
-- 
GitLab