1, What is metaprogramming
Metaprogramming is a technique for writing computer programs that can see themselves as data, so you can introspect, generate and / or modify it at run time.
Python provides introspection and real-time creation and modification capabilities for basic types such as functions and classes at the language level; We can use decorators to add additional functionality to existing functions, methods, or classes; At the same time, we can also change the behavior of classes by modifying some special methods;
2, Examples of use
- Facing a complex and changeable JSON data structure, although Python provides an API for processing JSON data, the return type is dict, which is very inconvenient and unfriendly; Next, an access method similar to the object hierarchy is realized through the meta programming ability provided by python;
import json
str = r'''{
"name":"mango",
"age": 30,
"address":{
"city":"beijing"
},
"schools":["xiaoxue","zhongxue"],
"sons":[
{
"name":"qing"
}
]
}'''
obj = json.loads(str)
print(type(obj))
print(obj.get('name'))
# <class 'dict'>
# mango
- Object oriented technology advocates encapsulating digital fields and controlling the verification of data fields through access controller; Next, it is implemented through the meta programming ability provided by python;
3, Pass__ getattr__ Acquisition of response dynamic fields
__ getattr__ Is an instance method, which is applicable to calling when accessing an undefined attribute, that is, when the attribute does not exist in the instance, the base class and the ancestor class of the corresponding class;
When obtaining the field value, we first check whether the corresponding field exists. If it does not exist, an exception will be thrown; If the field exists, detect the field type and decide whether to process the nested structure;
import json
from collections import abc
def loadJsonStr():
str = r'''{
"name":"mango",
"age": 30,
"address":{
"city":"beijing"
},
"schools":["xiaoxue","zhongxue"],
"sons":[
{
"name":"qing"
}
]
}'''
result = json.loads(str)
return result;
class JsonObject:
def __init__(self, jsondict):
self._data = dict(jsondict)
def __getattr__(self, name):
if name in self._data:
val = self._data.get(name)
if isinstance(val, abc.Mapping) or isinstance(val, abc.MutableSequence):
return self.initinner(val)
else:
return val
else:
raise AttributeError(f"{name} field does not exist")
def initinner(self, obj):
if isinstance(obj, abc.Mapping):
return self.__class__(obj)
elif isinstance(obj,abc.MutableSequence):
return [self.initinner(item) for item in obj]
else:
return obj
jobj = JsonObject(loadJsonStr())
print(jobj.name)
print(jobj.address)
print(jobj.address.city)
print(jobj.schools)
print(jobj.sons[0].name)
print(jobj.noField)
# mango
# <__main__.JsonObject object at 0x7ff7eac1cee0>
# beijing
# ['xiaoxue', 'zhongxue']
# qing
# AttributeError: noField field does not exist
5, Use__ new__ Create objects dynamically
We usually put__ init__ It's called a construction method, but it's actually used to build an instance__ new__: This is a class method that must return an instance. The returned instance is passed to the as the first parameter (i.e. self)__ init__ method. Because call__ init__ Method, and it is forbidden to return any value, so__ init__ Method is actually "initialization method". The real construction method is__ new__. We can complete the parsing of JSon field values in constructor China;
import json
from collections import abc
def loadJsonStr():
str = r'''{
"name":"mango",
"age": 30,
"address":{
"city":"beijing"
},
"schools":["xiaoxue","zhongxue"],
"sons":[
{
"name":"qing"
}
]
}'''
result = json.loads(str)
return result;
class JsonObject:
def __new__(cls, args, **kwargs):
obj = args
if isinstance(obj, abc.Mapping):
return super().__new__(cls)
elif isinstance(obj,abc.MutableSequence):
return [cls(item) for item in obj]
else:
return obj
def __init__(self, jsondict):
self._data = dict(jsondict)
def __getattr__(self, name):
if name in self._data:
val = self._data.get(name)
if isinstance(val, abc.Mapping) or isinstance(val, abc.MutableSequence):
return self.__class__(val)
else:
return val
else:
raise AttributeError(f"{name} field does not exist")
jobj = JsonObject(loadJsonStr())
print(jobj.name)
print(jobj.address)
print(jobj.address.city)
print(jobj.schools)
print(jobj.sons[0].name)
print(jobj.noField)
# mango
# <__main__.JsonObject object at 0x7ff7eac1cee0>
# beijing
# ['xiaoxue', 'zhongxue']
# qing
# AttributeError: noField field does not exist
6, Adding validation logic using the property decorator
We can use the property attribute provided by Python to add verification logic to the data field, so as to avoid the change of the caller; Although the property decorator is defined on the class, the compiler will first find it on the class. If it cannot be found, it will find it on the instance of the class;
import sys
class OrderItem:
def __init__(self, desc, count, price):
self.desc = desc
self.count = count
self.price = price
def subtotal(self):
return self.count * self.price
@property
def price(self):
print(f'{sys._getframe().f_code.co_name} getter')
return self._price
@price.setter
def price(self, val):
print(f'{sys._getframe().f_code.co_name} setter')
if val > 0:
self._price = val
else:
raise ValueError('price must be > 0')
@property
def count(self):
print(f'{sys._getframe().f_code.co_name} getter')
return self._count
@count.setter
def count(self, val):
print(f'{sys._getframe().f_code.co_name} setter')
if val > 0:
self._count = val
else:
raise ValueError('count must be > 0')
pbook = OrderItem('python books', 1, 50)
print(pbook.subtotal())
print(OrderItem.price)
print(OrderItem.price.setter)
print(OrderItem.price.getter)
print(vars(pbook))
jbook = OrderItem('java books', 0, 50)
print(jbook.subtotal())
# count setter
# price setter
# count getter
# price getter
# 50
# <property object at 0x7ffa8ddf8a90>
# <built-in method setter of property object at 0x7ffa8ddf8a90>
# <built-in method getter of property object at 0x7ffa8ddf8a90>
# {'desc': 'python books', '_count': 1, '_price': 50}
# count setter
# ValueError: count must be > 0
The logic for creating fields can be extracted as a common method
import sys
def buildVolidateField(name):
_name = f'_{name}'
def getter(obj):
return obj.__dict__.get(_name)
def setter(obj, value):
if value > 0:
obj.__dict__[_name]= value
else:
raise ValueError(f'{name} must be > 0')
return property(getter, setter)
class OrderItem:
price = buildVolidateField('price')
count = buildVolidateField('count')
def __init__(self, desc, count, price):
self.desc = desc
self.count = count
self.price = price
def subtotal(self):
return self.count * self.price
pbook = OrderItem('python books', 1, 50)
print(pbook.subtotal())
print(OrderItem.price)
print(OrderItem.price.setter)
print(OrderItem.price.getter)
print(vars(pbook))
jbook = OrderItem('java books', 0, 50)
print(jbook.subtotal())
# 50
# <property object at 0x7fbc90cfdd60>
# <built-in method setter of property object at 0x7fbc90cfdd60>
# <built-in method getter of property object at 0x7fbc90cfdd60>
# {'desc': 'python books', '_count': 1, '_price': 50}
# ValueError: count must be > 0
7, Using descriptor class to implement field verification logic
Descriptors are classes that implement specific protocols, including__ get__,__ set__ And__ delete__ method. The property class implements the complete descriptor protocol. Generally, only part of the protocol can be implemented. In fact, most descriptors we see in real code are only implemented__ get__ And__ set__ Methods, there are many, and only one of them is implemented.
import sys
class VolidateDescr:
def __init__(self, name):
self.name = f'_{name}'
def __set__(self, instance, value):
if value > 0:
instance.__dict__[self.name] = value
else:
raise ValueError(f'{self.name} must be > 0')
def __get__(self, instance, default):
if instance is None:
return self;
else:
return instance.__dict__[self.name]
class OrderItem:
price = VolidateDescr('price')
count = VolidateDescr('count')
def __init__(self, desc, count, price):
self.desc = desc
self.count = count
self.price = price
def subtotal(self):
return self.count * self.price
pbook = OrderItem('python books', 1, 50)
print(pbook.subtotal())
print(OrderItem.price)
print(OrderItem.price.__set__)
print(OrderItem.price.__get__)
print(vars(pbook))
jbook = OrderItem('java books', 0, 50)
print(jbook.subtotal())
# 50
# <__main__.VolidateDescr object at 0x7f162d0ac9a0>
# <bound method VolidateDescr.__set__ of <__main__.VolidateDescr object at 0x7f162d0ac9a0>>
# <bound method VolidateDescr.__get__ of <__main__.VolidateDescr object at 0x7f162d0ac9a0>>
# {'desc': 'python books', '_count': 1, '_price': 50}
# ValueError: _count must be > 0
At present, only two numeric fields have been added with verification. Next, add non null verification for desc string field; There is only verification difference between the two data types. We separate the verification logic from the field access control and implement two specific verification classes respectively;
import abc
class FieldDescr:
_countor = 0
def __init__(self):
self.name = f'_{self.__class__.__name__}_{self.__class__._countor}'
self.__class__._countor += 1
def __set__(self, instance, value):
setattr(instance, self.name, value)
def __get__(self, instance, owner):
if instance is None:
return self
else:
return getattr(instance, self.name)
class Validated(FieldDescr):
def __set__(self, instance, value):
value = self.validate(instance, value)
super().__set__(instance, value)
@abc.abstractmethod
def validate(self, instance, value):
'''this is abstract method'''
class GreatZeroIntField(Validated):
def validate(self, instance, value):
if value <= 0:
raise ValueError(f'{self.name} value must be > 0')
return value
class NoEmptyStrField(Validated):
def validate(self, instance, value):
value = value.strip()
if len(value) == 0:
raise ValueError('value cant not be empty or blank')
return value
class OrderItem:
descr = NoEmptyStrField()
price = GreatZeroIntField()
count = GreatZeroIntField()
def __init__(self, descr, price, count):
self.descr = descr
self.price = price
self.count = count
def subtotal(self):
return self.count * self.price
pbook = OrderItem('python books', 1, 50)
print(pbook.subtotal())
print(OrderItem.price)
print(OrderItem.price.__set__)
print(OrderItem.price.__get__)
print(vars(pbook))
jbook = OrderItem('java books', 0, 50)
print(jbook.subtotal())
# 50
# <__main__.GreatZeroIntField object at 0x7fa2eb37fd00>
# <bound method Validated.__set__ of <__main__.GreatZeroIntField object at 0x7fa2eb37fd00>>
# <bound method FieldDescr.__get__ of <__main__.GreatZeroIntField object at 0x7fa2eb37fd00>>
# {'_NoEmptyStrField_0': 'python books', '_GreatZeroIntField_0': 1, '_GreatZeroIntField_1': 50}
# ValueError: _GreatZeroIntField_0 value must be > 0
- Name of custom data field
Up to now, we have encapsulated the automatic generation feature, and the name of the automatically generated data field does not correspond well to the corresponding feature name on the class; Next, the name of the data field is customized through the class decorator and metaclass;
The class decorator is executed after the compiler compiles the class. At this time, the features on the class have been generated, and we can traverse the properties of the class__ dict__, Find the corresponding property and modify the value of its name field;
import abc
def renamePrivateField(cls):
for key,value in cls.__dict__.items():
if isinstance(value, Validated):
value.name = f'_{value.__class__.__name__}_{key}'
return cls
class FieldDescr:
_countor = 0
def __init__(self):
self.name = f'_{self.__class__.__name__}_{self.__class__._countor}'
self.__class__._countor += 1
def __set__(self, instance, value):
setattr(instance, self.name, value)
def __get__(self, instance, owner):
if instance is None:
return self
else:
return getattr(instance, self.name)
class Validated(FieldDescr):
def __set__(self, instance, value):
value = self.validate(instance, value)
super().__set__(instance, value)
@abc.abstractmethod
def validate(self, instance, value):
'''this is abstract method'''
class GreatZeroIntField(Validated):
def validate(self, instance, value):
if value <= 0:
raise ValueError(f'{self.name} value must be > 0')
return value
class NoEmptyStrField(Validated):
def validate(self, instance, value):
value = value.strip()
if len(value) == 0:
raise ValueError('value cant not be empty or blank')
return value
@renamePrivateField
class OrderItem:
descr = NoEmptyStrField()
price = GreatZeroIntField()
count = GreatZeroIntField()
def __init__(self, descr, price, count):
self.descr = descr
self.price = price
self.count = count
def subtotal(self):
return self.count * self.price
pbook = OrderItem('python books', 1, 50)
print(pbook.subtotal())
print(OrderItem.price)
print(OrderItem.price.name)
print(OrderItem.price.__set__)
print(OrderItem.price.__get__)
print(vars(pbook))
# 50
# <__main__.GreatZeroIntField object at 0x7f23e67bf2b0>
# _GreatZeroIntField_price
# <bound method Validated.__set__ of <__main__.GreatZeroIntField object at 0x7f23e67bf2b0>>
# <bound method FieldDescr.__get__ of <__main__.GreatZeroIntField object at 0x7f23e67bf2b0>>
# {'_NoEmptyStrField_descr': 'python books', '_GreatZeroIntField_price': 1, '_GreatZeroIntField_count': 50}
Because the class decorator is directly executed after the class is compiled, it may be overwritten by subclasses. Metaclasses can solve this problem
import abc
class FieldDescr:
_countor = 0
def __init__(self):
self.name = f'_{self.__class__.__name__}_{self.__class__._countor}'
self.__class__._countor += 1
def __set__(self, instance, value):
setattr(instance, self.name, value)
def __get__(self, instance, owner):
if instance is None:
return self
else:
return getattr(instance, self.name)
class Validated(FieldDescr):
def __set__(self, instance, value):
value = self.validate(instance, value)
super().__set__(instance, value)
@abc.abstractmethod
def validate(self, instance, value):
'''this is abstract method'''
class GreatZeroIntField(Validated):
def validate(self, instance, value):
if value <= 0:
raise ValueError(f'{self.name} value must be > 0')
return value
class NoEmptyStrField(Validated):
def validate(self, instance, value):
value = value.strip()
if len(value) == 0:
raise ValueError('value cant not be empty or blank')
return value
class renamePrivateFieldMeta(type):
def __init__(cls, name, bases, attr_dict):
super().__init__(name, bases, attr_dict)
for key, value in cls.__dict__.items():
if isinstance(value, Validated):
value.name = f'_{value.__class__.__name__}_{key}'
class OrderEntity(metaclass=renamePrivateFieldMeta):
'''rename entity'''
class OrderItem(OrderEntity):
descr = NoEmptyStrField()
price = GreatZeroIntField()
count = GreatZeroIntField()
def __init__(self, descr, price, count):
self.descr = descr
self.price = price
self.count = count
def subtotal(self):
return self.count * self.price
pbook = OrderItem('python books', 1, 50)
print(pbook.subtotal())
print(OrderItem.price)
print(OrderItem.price.name)
print(OrderItem.price.__set__)
print(OrderItem.price.__get__)
print(vars(pbook))
# 50
# <__main__.GreatZeroIntField object at 0x7f393be8c070>
# _GreatZeroIntField_price
# <bound method Validated.__set__ of <__main__.GreatZeroIntField object at 0x7f393be8c070>>
# <bound method FieldDescr.__get__ of <__main__.GreatZeroIntField object at 0x7f393be8c070>>
# {'_NoEmptyStrField_descr': 'python books', '_GreatZeroIntField_price': 1, '_GreatZeroIntField_count': 50}