Order-preserving CommonDictionaryStorage

Src/IronPython/Runtime/DictionaryOps.cs

@@ -6,16 +6,14 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Runtime.CompilerServices;
+using System.Linq;
 using System.Text;
 
-using Microsoft.Scripting;
-using Microsoft.Scripting.Runtime;
-
-using IronPython.Runtime.Binding;
 using IronPython.Runtime.Operations;
 using IronPython.Runtime.Types;
 
+using Microsoft.Scripting.Runtime;
+
 namespace IronPython.Runtime {
     /// <summary>
     /// Provides both helpers for implementing Python dictionaries as well
@@ -100,14 +98,14 @@ public static object pop(PythonDictionary self, object key, object defaultValue)
         }
 
         public static PythonTuple popitem(PythonDictionary self) {
-            using IEnumerator<KeyValuePair<object, object>> ie = self.GetEnumerator();
-            if (ie.MoveNext()) {
-                object key = ie.Current.Key;
-                object val = ie.Current.Value;
-                self.RemoveDirect(key);
-                return PythonTuple.MakeTuple(key, val);
+            try {
+                var pair = self._storage.GetItems().Last();
+                self.RemoveDirect(pair.Key);
+                return PythonTuple.MakeTuple(pair.Key, pair.Value);
+            }
+            catch (InvalidOperationException) {
+                throw PythonOps.KeyError("dictionary is empty");
             }
-            throw PythonOps.KeyError("dictionary is empty");
         }
 
         public static object setdefault(PythonDictionary self, object key) {
@@ -122,7 +120,11 @@ public static object setdefault(PythonDictionary self, object key, object defaul
 
         public static void update(CodeContext/*!*/ context, PythonDictionary/*!*/ self, object other) {
             if (other is PythonDictionary pyDict) {
-                pyDict._storage.CopyTo(ref self._storage);
+                if (pyDict.TreatAsMapping) {
+                    SlowUpdate(context, self, other);
+                } else {
+                    pyDict._storage.CopyTo(ref self._storage);
+                }
             } else {
                 SlowUpdate(context, self, other);
             }
@@ -131,7 +133,7 @@ public static void update(CodeContext/*!*/ context, PythonDictionary/*!*/ self,
         private static void SlowUpdate(CodeContext/*!*/ context, PythonDictionary/*!*/ self, object other) {
             if (other is MappingProxy proxy) {
                 update(context, self, proxy.GetDictionary(context));
-            } else if (other is IDictionary dict) {
+            } else if (other is IDictionary dict && other is not PythonDictionary) {
                 IDictionaryEnumerator e = dict.GetEnumerator();
                 while (e.MoveNext()) {
                     self._storage.Add(ref self._storage, e.Key, e.Value);

Src/IronPython/Runtime/DictionaryStorage.cs

@@ -8,7 +8,6 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 
-using Microsoft.Scripting;
 using Microsoft.Scripting.Runtime;
 
 namespace IronPython.Runtime {
@@ -117,8 +116,7 @@ public virtual DictionaryStorage Clone() {
             return storage;
         }
 
-        public virtual void EnsureCapacityNoLock(int size) {
-        }
+        public virtual void EnsureCapacityNoLock(int size) { }
 
         public virtual IEnumerator<KeyValuePair<object?, object?>> GetEnumerator() {
             return GetItems().GetEnumerator();

Src/IronPython/Runtime/EmptyDictionaryStorage.cs

@@ -1,6 +1,11 @@
-using System;
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using System;
 using System.Collections.Generic;
-using System.Text;
 
 using IronPython.Runtime.Operations;
 
@@ -12,10 +17,9 @@ namespace IronPython.Runtime {
     internal class EmptyDictionaryStorage : DictionaryStorage {
         public static EmptyDictionaryStorage Instance = new EmptyDictionaryStorage();
 
-        private EmptyDictionaryStorage() {
-        }
+        private EmptyDictionaryStorage() { }
 
-        public override void Add(ref DictionaryStorage storage, object key, object value) {
+        public override void Add(ref DictionaryStorage storage, object? key, object? value) {
             lock (this) {
                 if (storage == this) {
                     CommonDictionaryStorage newStorage = new CommonDictionaryStorage();
@@ -24,12 +28,12 @@ public override void Add(ref DictionaryStorage storage, object key, object value
                     return;
                 }
             }
-            
+
             // race, try again...
             storage.Add(ref storage, key, value);
         }
 
-        public override bool Remove(ref DictionaryStorage storage, object key) {
+        public override bool Remove(ref DictionaryStorage storage, object? key) {
             return false;
         }
 
@@ -44,18 +48,17 @@ public override DictionaryStorage AsMutable(ref DictionaryStorage storage) {
             return storage.AsMutable(ref storage);
         }
 
-        public override void Clear(ref DictionaryStorage storage) {
-        }
+        public override void Clear(ref DictionaryStorage storage) { }
 
-        public override bool Contains(object key) {
+        public override bool Contains(object? key) {
             // make sure argument is valid, do not calculate hash
             if (PythonContext.IsHashable(key)) {
                 return false;
             }
             throw PythonOps.TypeErrorForUnhashableObject(key);
         }
 
-        public override bool TryGetValue(object key, out object value) {
+        public override bool TryGetValue(object? key, out object? value) {
             value = null;
             return false;
         }
@@ -64,8 +67,8 @@ public override int Count {
             get { return 0; }
         }
 
-        public override List<KeyValuePair<object, object>> GetItems() {
-            return new List<KeyValuePair<object, object>>();
+        public override List<KeyValuePair<object?, object?>> GetItems() {
+            return new List<KeyValuePair<object?, object?>>();
         }
 
         public override DictionaryStorage Clone() {

Src/IronPython/Runtime/Operations/PythonOps.cs

@@ -2614,6 +2614,8 @@ public static PythonDictionary UserMappingToPythonDictionary(CodeContext/*!*/ co
         }
 
         public static PythonDictionary CopyAndVerifyPythonDictionary(PythonFunction function, PythonDictionary dict) {
+            if (dict.TreatAsMapping) return CopyAndVerifyUserMapping(function, dict);
+
             if (dict._storage.HasNonStringAttributes()) {
                 throw TypeError("{0}() keywords must be strings", function.__name__);
             }

Src/IronPython/Runtime/PythonDictionary.cs

@@ -151,11 +151,7 @@ public bool TryGetValue(object key, out object value) {
             // we need to manually look up a slot to get the correct behavior when
             // the __missing__ function is declared on a sub-type which is an old-class
             if (GetType() != typeof(PythonDictionary) &&
-                PythonTypeOps.TryInvokeBinaryOperator(DefaultContext.Default,
-                this,
-                key,
-                "__missing__",
-                out value)) {
+                PythonTypeOps.TryInvokeBinaryOperator(DefaultContext.Default, this, key, "__missing__", out value)) {
                 return true;
             }
 
@@ -353,8 +349,7 @@ public object setdefault(object key, object defaultValue) {
         public DictionaryValueView values() => new DictionaryValueView(this);
 
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic")]
-        public void update() {
-        }
+        public void update() { }
 
         public void update(CodeContext/*!*/ context, [ParamDictionary] IDictionary<object, object> other\u00F8) {
             DictionaryOps.update(context, this, other\u00F8);
@@ -683,6 +678,11 @@ internal bool TryRemoveValue(object key, out object value) {
             return _storage.TryRemoveValue(ref _storage, key, out value);
         }
 
+        /// <summary>
+        /// If __iter__ is overridden then we should treat the dict as a mapping.
+        /// </summary>
+        internal bool TreatAsMapping => GetType() != typeof(PythonDictionary) && ((Func<object>)__iter__).Method.DeclaringType != typeof(PythonDictionary);
+
         #region Debugger View
 
         internal class DebugProxy {

Src/IronPythonTest/Cases/CPythonCasesManifest.ini

@@ -207,9 +207,6 @@ Ignore=true
 RunCondition=$(IS_DEBUG) # https://github.com/IronLanguages/ironpython3/issues/1067
 Timeout=600000 # 10 minute timeout
 
-[CPython.test_call] # IronPython.test_call_stdlib
-Ignore=true
-
 [CPython.test_capi]
 Ignore=true
 Reason=ImportError: No module named _testcapi

Tests/test_functools_stdlib.py

@@ -43,10 +43,8 @@ def load_tests(loader, standard_tests, pattern):
         ]
 
         skip_tests = [
-            test.test_functools.TestLRUC('test_kwargs_order'), # intermittent failures - https://github.com/IronLanguages/ironpython3/issues/1460
             test.test_functools.TestLRUC('test_lru_cache_threaded2'), # intermittent failures
             test.test_functools.TestLRUC('test_lru_cache_threaded3'), # intermittent failures
-            test.test_functools.TestLRUPy('test_kwargs_order'), # intermittent failures - https://github.com/IronLanguages/ironpython3/issues/1460
             test.test_functools.TestLRUPy('test_lru_cache_threaded2'), # intermittent failures
             test.test_functools.TestLRUPy('test_lru_cache_threaded3'), # intermittent failures
             test.test_functools.TestPartialC('test_recursive_pickle'), # StackOverflowException

Tests/test_ordered_dict_stdlib.py

@@ -15,40 +15,22 @@ def load_tests(loader, standard_tests, pattern):
 
     if is_ironpython:
         failing_tests = [
-            test.test_ordered_dict.CPythonOrderedDictSubclassTests('test_468'), # https://github.com/IronLanguages/ironpython3/issues/1460
             test.test_ordered_dict.CPythonOrderedDictSubclassTests('test_copying'), # pickle.PicklingError
             test.test_ordered_dict.CPythonOrderedDictSubclassTests('test_free_after_iterating'), # AssertionError
             test.test_ordered_dict.CPythonOrderedDictSubclassTests('test_iterators_pickling'), # pickle.PicklingError
             test.test_ordered_dict.CPythonOrderedDictSubclassTests('test_key_change_during_iteration'), # AssertionError: RuntimeError not raised
             test.test_ordered_dict.CPythonOrderedDictSubclassTests('test_pickle_recursive'), # pickle.PicklingError
-            test.test_ordered_dict.CPythonOrderedDictTests('test_468'), # https://github.com/IronLanguages/ironpython3/issues/1460
             test.test_ordered_dict.CPythonOrderedDictTests('test_free_after_iterating'), # AssertionError
             test.test_ordered_dict.CPythonOrderedDictTests('test_iterators_pickling'), # pickle.PicklingError
             test.test_ordered_dict.CPythonOrderedDictTests('test_key_change_during_iteration'), # AssertionError: RuntimeError not raised
-            test.test_ordered_dict.PurePythonOrderedDictSubclassTests('test_468'), # https://github.com/IronLanguages/ironpython3/issues/1460
             test.test_ordered_dict.PurePythonOrderedDictSubclassTests('test_copying'), # pickle.PicklingError
             test.test_ordered_dict.PurePythonOrderedDictSubclassTests('test_free_after_iterating'), # AssertionError
             test.test_ordered_dict.PurePythonOrderedDictSubclassTests('test_pickle_recursive'), # pickle.PicklingError
-            test.test_ordered_dict.PurePythonOrderedDictTests('test_468'), # https://github.com/IronLanguages/ironpython3/issues/1460
             test.test_ordered_dict.PurePythonOrderedDictTests('test_free_after_iterating'), # AssertionError
         ]
 
         skip_tests = [
-            # https://github.com/IronLanguages/ironpython3/issues/1556
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_abc'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_clear'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_delitem'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_delitem_hash_collision'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_detect_deletion_during_iteration'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_highly_nested'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_highly_nested_subclass'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_init'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_override_update'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_popitem'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_reinsert'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_setitem'),
-            test.test_ordered_dict.CPythonBuiltinDictTests('test_update'),
-
+            test.test_ordered_dict.CPythonBuiltinDictTests('test_highly_nested_subclass'), # intermittent AssertionError - GC related?
             test.test_ordered_dict.CPythonOrderedDictSubclassTests('test_highly_nested_subclass'), # intermittent AssertionError - GC related?
             test.test_ordered_dict.CPythonOrderedDictTests('test_highly_nested_subclass'), # intermittent AssertionError - GC related?
             test.test_ordered_dict.PurePythonOrderedDictSubclassTests('test_highly_nested_subclass'), # intermittent AssertionError - GC related?