nacos Distributed Configuration Center
Personal Alibaba nacos Renovation Project: alibaba_nacos
1. Service Startup Configuration Loading
Source: NacosPropertySourceLocator
Workflow: Nacos customizes the PropertySourceLocator to get data from the configuration center at service startup, then add and re-run the local environment to load the configuration
1.0 Configuration Loading Logic
Method:com.alibaba.cloud.nacos.client.NacosPropertySourceLocator.locate
@Override public PropertySource<?> locate(Environment env) { // Create ConfigService with NACOS attribute configuration to configure the central interactive API ConfigService configService = nacosConfigProperties.configServiceInstance(); if (null == configService) { log.warn("no instance of config service found, can't load config from nacos"); return null; } // Create NACOS Attribute Source Builder nacosPropertySourceBuilder = new NacosPropertySourceBuilder(configService, nacosConfigProperties.getTimeout()); // Define DataID prefix with priority greater than NACOS configuration name String name = nacosConfigProperties.getName(); String dataIdPrefix = nacosConfigProperties.getPrefix(); if (StringUtils.isEmpty(dataIdPrefix)) { dataIdPrefix = name; } // Configuration name and dataId prefix do not explicitly specify defaultSpring.application.name if (StringUtils.isEmpty(dataIdPrefix)) { dataIdPrefix = env.getProperty("spring.application.name"); } // Construct an attribute source collection class to hold different configurations CompositePropertySource composite = new CompositePropertySource(NACOS_PROPERTY_SOURCE_NAME); //NAOCOS configuration load order: Shared configuration --> Extended configuration --> Self configuration (higher priority later) loadSharedConfiguration(composite); loadExtConfiguration(composite); loadApplicationConfiguration(composite, dataIdPrefix, nacosConfigProperties, env); return composite; }
nacos Configuration Client Reference Configuration:
server: port: 8081 spring: profiles: active: dev application: name: nacos_config_client cloud: nacos: config: server-addr: localhost:8848 namespace: ${spring.profiles.active} # Namespace, default public, note that the namespace ID must be group: ${spring.application.name} prefix: ${spring.application.name} # File prefix file-extension: properties # file extension username: iot-app password: iot-app ## Note: The shared dataID group name must be DEFAULT_GROUP sharedDataIds: shared.properties refreshableDataIds: shared.properties extConfig: - { dataId: common.properties, group: common, refresh: true }
1.1 Load Shared Configuration
/** * Load Shared Configuration * @param compositePropertySource */ private void loadSharedConfiguration(CompositePropertySource compositePropertySource) { String sharedDataIds = nacosConfigProperties.getSharedDataIds(); String refreshDataIds = nacosConfigProperties.getRefreshableDataIds(); if (sharedDataIds == null || sharedDataIds.trim().length() == 0) { return; } String[] sharedDataIdArray = sharedDataIds.split(SHARED_CONFIG_SEPARATOR_CHAR); checkDataIdFileExtension(sharedDataIdArray); for (String dataId : sharedDataIdArray) { String fileExtension = dataId.substring(dataId.lastIndexOf(".") + 1); boolean isRefreshable = checkDataIdIsRefreshable(refreshDataIds, dataId); // Indicates shared configuration GroupID default DEFAULT_GROUP loadNacosDataIfPresent(compositePropertySource, dataId, "DEFAULT_GROUP", fileExtension, isRefreshable); } }
1.2 Load Extension Configuration
/** * Load NACOS Extension Configuration * @param compositePropertySource */ private void loadExtConfiguration(CompositePropertySource compositePropertySource) { List<NacosConfigProperties.Config> extConfigs = nacosConfigProperties.getExtConfig(); if (CollectionUtils.isEmpty(extConfigs)) { return; } checkExtConfiguration(extConfigs); for (NacosConfigProperties.Config config : extConfigs) { String dataId = config.getDataId(); String fileExtension = dataId.substring(dataId.lastIndexOf(DOT) + 1); loadNacosDataIfPresent(compositePropertySource, dataId, config.getGroup(),fileExtension, config.isRefresh()); } }
1.3 Load Project Private Configuration
private void loadApplicationConfiguration(CompositePropertySource compositePropertySource, String dataIdPrefix, NacosConfigProperties properties, Environment environment) { String fileExtension = properties.getFileExtension(); String nacosGroup = properties.getGroup(); // load directly once by default loadNacosDataIfPresent(compositePropertySource, dataIdPrefix, nacosGroup, fileExtension, true); // load with suffix, which have a higher priority than the default loadNacosDataIfPresent(compositePropertySource, dataIdPrefix + DOT + fileExtension, nacosGroup, fileExtension, true); // Loaded with profile, which have a higher priority than the suffix for (String profile : environment.getActiveProfiles()) { String dataId = dataIdPrefix + SEP1 + profile + DOT + fileExtension; loadNacosDataIfPresent(compositePropertySource, dataId, nacosGroup, fileExtension, true); } }
1.4 Bottom Load Configuration Logic
private void loadNacosDataIfPresent(final CompositePropertySource composite, final String dataId, final String group, String fileExtension, boolean isRefreshable) { if (null == dataId || dataId.trim().length() < 1) { return; } if (null == group || group.trim().length() < 1) { return; } // Create Nacos Configuration Source NacosPropertySource propertySource = this.loadNacosPropertySource(dataId, group, fileExtension, isRefreshable); // The addFirstPropertySource method is called by the subsequent load in the order before and after loading, indicating priority private configuration > extended configuration > shared configuration this.addFirstPropertySource(composite, propertySource, false); }
Loading Nacos configuration data
private NacosPropertySource loadNacosPropertySource(final String dataId, final String group, String fileExtension, boolean isRefreshable) { if (NacosContextRefresher.getRefreshCount() != 0) { // Auto-fetch return from cache if auto-refresh configuration is not supported if (!isRefreshable) { return NacosPropertySourceRepository.getNacosPropertySource(dataId); } } //Constructor retrieves data from the configuration center return nacosPropertySourceBuilder.build(dataId, group, fileExtension, isRefreshable); } //Bottom Interface:com.alibaba.cloud.nacos.client.NacosPropertySourceBuilder.loadNacosData, configuration data caches NacosPropertySourceRepository private Properties loadNacosData(String dataId, String group, String fileExtension) { String data = null; try { // http remote access configuration center, get configuration data data = configService.getConfig(dataId, group, timeout); if (StringUtils.isEmpty(data)) { log.warn("Ignore the empty nacos configuration and get it based on dataId[{}] & group[{}]",dataId, group); return EMPTY_PROPERTIES; } log.info(String.format("Loading nacos data, dataId: '%s', group: '%s', data: %s", dataId, group, data)); Properties properties = NacosDataParserHandler.getInstance().parseNacosData(data, fileExtension); return properties == null ? EMPTY_PROPERTIES : properties; } catch (NacosException e) { log.error("get data from Nacos error,dataId:{}, ", dataId, e); } catch (Exception e) { log.error("parse data from Nacos error,dataId:{},data:{},", dataId, data, e); } return EMPTY_PROPERTIES; }
2. Configure hot load after service startup
Hot load configuration: @RefreshScope(spring remote support), @NacosValue(nacos support)
Difference:
- @RefreshScope: Bean is cleared, accessed again, and created
- @NacosValue: Modify NacosValue-modified Bean properties directly through a reflection mechanism
2.1 Http Long Polling for Configuration Real-Time Synchronization
Interface: LongPollingRunnable
private class LongPollingRunnable implements Runnable { private final int taskId; public LongPollingRunnable(int taskId) { this.taskId = taskId; } @Override public void run() { List<CacheData> localCacheDataList = new ArrayList<>(); List<String> inInitializingCacheList = new ArrayList<>(); try { // check failover config: Check the JVM and local profile cache for (CacheData localCacheData : cacheMap.get().values()) { if (localCacheData.getTaskId() == taskId) { localCacheDataList.add(localCacheData); try { syncDiskCacheToJvmCacheConfig(localCacheData); //Check MD5 using local cache profile data, synchronize cache MD5 to listener if (localCacheData.isUseLocalConfigInfo()) { localCacheData.checkListenerMd5(); } } catch (Exception e) { log.error("get local config info error", e); } } } // check server config: Compare local cache MD5 with server configuration data to confirm the changed cache coordinates again:Shared.properties+DEFAULT_GROUP+dev (Long Polling Check) // Long Polling: The nacos client normally sends configuration check requests every 30 seconds. If the configuration center finds that the client and server configurations are inconsistent (configuration update), it receives the request and returns immediately. If it is consistent, it returns normal response for 30 seconds // Note: Considering network factors, the client Http timeout should be slightly longer than 30 seconds and the server response should be slightly shorter than 30 seconds. Otherwise, due to network reasons, the server responds normally for 30 seconds and network latency results in the client failing to get response timeout error // Code: long readTimeoutMs = timeout + (long)Math.round(timeout >> 1); timeout = normal polling time* 1.5 List<String> changedGroupKeys = checkUpdateDataIds(localCacheDataList, inInitializingCacheList); log.info("get changedGroupKeys ===> {}", changedGroupKeys); //Get change configuration information from nacos server for (String groupKey : changedGroupKeys) { String[] key = GroupKey.parseKey(groupKey); String dataId = key[0]; String group = key[1]; String tenant = null; if (key.length == 3) { tenant = key[2]; } try { String[] ct = getServerConfig(dataId, group, tenant, 6000L); //Update local cache CacheData localCacheData = cacheMap.get().get(GroupKey.getKeyTenant(dataId, group, tenant)); localCacheData.setContent(ct[0]); if (null != ct[1]) { localCacheData.setType(ct[1]); } log.info("[{}] [data-received] dataId={}, group={}, tenant={}, md5={}, content={}, type={}", httpAgent.getName(), dataId, group, tenant, localCacheData.getMd5(), ContentUtils.truncateContent(ct[0]), ct[1]); } catch (NacosException ioe) { String message = String.format( "[%s] [get-update] get changed config exception. dataId=%s, group=%s, tenant=%s", httpAgent.getName(), dataId, group, tenant); log.error(message, ioe); } } for (CacheData cacheData : localCacheDataList) { //Not initialized, or initialized but not using local file cache to configure data if (!cacheData.isInitializing() || inInitializingCacheList.contains(GroupKey.getKeyTenant(cacheData.dataId, cacheData.group, cacheData.tenant))) { // Check the local listener cache after updating data from the configuration center, mainly to synchronously update Spring property information cacheData.checkListenerMd5(); cacheData.setInitializing(false); } } inInitializingCacheList.clear(); executorService.execute(this); } catch (Throwable e) { // If the rotation training task is abnormal, the next execution time of the task will be punished log.error("longPolling error : ", e); executorService.schedule(this, taskPenaltyTime, TimeUnit.MILLISECONDS); } } }
2.2 Cache listener:cacheData.checkListenerMd5();
Note: A cache corresponds to one listener, and a profile corresponds to one listener listening process
Cache configuration listener registration interface:com.alibaba.cloud.nacos.refresh.NacosContextRefresher.registerNacosListener
Bottom Code:
private void safeNotifyListener(final String dataId, final String group, final String content, final String type, final String md5, final ManagerListenerWrap listenerWrap) { final ConfigListener configListener = listenerWrap.configListener; Runnable job = () -> { ClassLoader myClassLoader = Thread.currentThread().getContextClassLoader(); ClassLoader appClassLoader = configListener.getClass().getClassLoader(); try { //Shared Configuration Listener if (configListener instanceof AbstractSharedConfigListener) { AbstractSharedConfigListener adapter = (AbstractSharedConfigListener) configListener; adapter.fillContext(dataId, group); log.info("[{}] [notify-context] dataId={}, group={}, md5={}", name, dataId, group, md5); } // Set the thread classloader to the specific webapp's classloader before executing the callback to avoid an exception or misuse of invoking the spi interface in the callback method (this can happen with multiple application deployments). Thread.currentThread().setContextClassLoader(appClassLoader); ConfigResponse configResponse = new ConfigResponse(); configResponse.setDataId(dataId); configResponse.setGroup(group); configResponse.setContent(content); configFilterChainManager.doFilter(null, configResponse); String tempContent = configResponse.getContent(); //Callback listener gets configuration information and sends RefreshEvent notification to Spring to refresh the configuration, which is equivalent to reloading the configuration configListener.receiveConfigInfo(tempContent); // compare lastContent and content: Customize NacosValue support listeners if (configListener instanceof AbstractConfigChangeConfigListener) { Map data = ConfigChangeHandler.getInstance().parseChangeData(listenerWrap.lastContent, content, type); ConfigChangeEvent configChangeEvent = new ConfigChangeEvent(data); ((AbstractConfigChangeConfigListener) configListener).receiveConfigChange(configChangeEvent); listenerWrap.lastContent = content; } // Update Configuration Listener MD5 Value listenerWrap.lastCallMd5 = md5; log.info("[{}] [notify-ok] dataId={}, group={}, md5={}, listener={} ", name, dataId, group, md5, configListener); } catch (NacosException nacosException) { log.error("[{}] [notify-error] dataId={}, group={}, md5={}, listener={} errCode={} errMsg={}", name, dataId, group, md5, configListener, nacosException.getErrCode(), nacosException.getErrMsg()); } catch (Throwable t) { log.error("[{}] [notify-error] dataId={}, group={}, md5={}, listener={} tx={}", name, dataId, group, md5, configListener, t.getCause()); } finally { Thread.currentThread().setContextClassLoader(myClassLoader); } }; final long startNotify = System.currentTimeMillis(); try { //If the listener maintains the thread pool, it is handed over to the thread pool to run, otherwise it runs synchronously if (null != configListener.getExecutor()) { configListener.getExecutor().execute(job); } else { job.run(); } } catch (Throwable t) { log.error("[{}] [notify-error] dataId={}, group={}, md5={}, listener={} throwable={}", name, dataId, group, md5, configListener, t.getCause()); } final long finishNotify = System.currentTimeMillis(); log.info("[{}] [notify-listener] time cost={}ms in ClientWorker, dataId={}, group={}, md5={}, listener={} ", name, (finishNotify - startNotify), dataId, group, md5, configListener); }
2.3 Monitor implementation configuration hot update: @RefreshScope
RefreshScope: Custom Bean's workspace. spring defaults to singletons, prototypes, and supports extended Bean Scope, one of which is RefreshScope
RefreshScope effect: dynamic refresh during Bean run
// Default Anonymous Listener ConfigListener configListener = listenerMap.computeIfAbsent(dataId, i -> new ConfigListener() { @Override public void receiveConfigInfo(String configInfo) { refreshCountIncrement(); String md5 = ""; if (!StringUtils.isEmpty(configInfo)) { try { MessageDigest md = MessageDigest.getInstance("MD5"); md5 = new BigInteger(1, md.digest(configInfo.getBytes(StandardCharsets.UTF_8))).toString(16); } catch (NoSuchAlgorithmException e) { log.warn("[Nacos] unable to get md5 for dataId: " + dataId, e); } } refreshHistory.add(dataId, md5); //Publish refresh events applicationContext.publishEvent(new RefreshEvent(this, null, "Refresh Nacos config")); if (log.isDebugEnabled()) { log.debug("Refresh Nacos config group " + group + ",dataId" + dataId); } } @Override public Executor getExecutor() { return null; } });
RefreshEvent event listening processing:org.springframework.cloud.endpoint.event.RefreshEventListener.onApplicationEvent
// Refresh event handling public void handle(RefreshEvent event) { if (this.ready.get()) { // don't handle events before app is ready log.debug("Event received " + event.getEventDesc()); // Refresh specific logic Set<String> keys = this.refresh.refresh(); log.info("Refresh keys changed: " + keys); } } //org.springframework.cloud.context.refresh.ContextRefresher.refresh public synchronized Set<String> refresh() { // Refresh configuration, equivalent to NacosPropertySourceLocator retrieving data from Nacos Configuration Center Set<String> keys = refreshEnvironment(); //refreshScope Refresh this.scope.refreshAll(); return keys; } //this.scope.refreshAll(); //org.springframework.cloud.context.scope.refresh.RefreshScope.refreshAll public void refreshAll() { super.destroy(); // Publish RefreshScopeRefreshedEvent events, spring defaults to no listener and can listen on special business processes, such as Eureka this.context.publishEvent(new RefreshScopeRefreshedEvent()); } //Parent destroy:org.springframework.cloud.context.scope.GenericScope.destroy() @Override public void destroy() { List<Throwable> errors = new ArrayList<Throwable>(); // Clear Bean Cache, equivalent to clearing all Beans managed by RefreshScope Collection<BeanLifecycleWrapper> wrappers = this.cache.clear(); for (BeanLifecycleWrapper wrapper : wrappers) { try { Lock lock = this.locks.get(wrapper.getName()).writeLock(); lock.lock(); try { wrapper.destroy(); } finally { lock.unlock(); } } catch (RuntimeException e) { errors.add(e); } } if (!errors.isEmpty()) { throw wrapIfNecessary(errors.get(0)); } this.errors.clear(); }
RefreshScope Bean Get Logic
@Override public Object get(String name, ObjectFactory<?> objectFactory) { // Cache holds RefreshScope and objectFactory (including Bean Creation Method and Bean Instance Cache) BeanLifecycleWrapper value = this.cache.put(name, new BeanLifecycleWrapper(name, objectFactory)); this.locks.putIfAbsent(name, new ReentrantReadWriteLock()); try { return value.getBean(); } catch (RuntimeException e) { this.errors.put(name, e); throw e; } } // objectFactory gets RefreshScope Bean, returns directly if Bean Cache exists, and does not exist to be recreated following the Spring Bean creation process public Object getBean() { if (this.bean == null) { synchronized (this.name) { if (this.bean == null) { this.bean = this.objectFactory.getObject(); } } } return this.bean; }
Summary: @RefreshScope configuration hot load logic: publish FreshEvent events, refresh nacos configuration, empty RefreshScope scope beans, create when accessing beans again (delete then create)
2.3 Monitor implementation configuration hot update: @NacosValue
How it works: Direct modification of the value (reflection mechanism) of a member variable of a conservative Bean is more efficient than @RefreshScope
Monitor:com.alibaba.cloud.nacos.refresh.NacosContextRefresher.registerNacosListener
Workflow: Send ConfigChangeEvent events when configuration changes
AbstractConfigChangeConfigListener abstractConfigChangeConfigListener = new AbstractConfigChangeConfigListener() { @Override public void receiveConfigChange(ConfigChangeEvent event) { applicationContext.publishEvent(event); } };
ConfigChangeEvent event listener: NacosValueAnnotationBeanPostProcessor
@Override public void onApplicationEvent(ConfigChangeEvent event) { // In to this event receiver, the environment has been updated the // latest configuration information, pull directly from the environment // fix issue #142 // Placeholder NacosValueTargetMap: NacosValueAnnotationBeanPostProcessor implements the BeanPostProcessor interface, intercepts beans identifying the @NacosValue property before Bean initialization and caches them in placeholder NacosValueTargetMap for (Map.Entry<String, List<NacosValueTarget>> entry : placeholderNacosValueTargetMap.entrySet()) { String key = environment.resolvePlaceholders(entry.getKey()); String newValue = environment.getProperty(key); if (newValue == null) { continue; } List<NacosValueTarget> beanPropertyList = entry.getValue(); for (NacosValueTarget target : beanPropertyList) { String md5String = MD5.getInstance().getMD5String(newValue); // Comparing the old and new MD5 values of the variables identified by @NacosValue indicates that the scalar values need to be updated boolean isUpdate = !target.lastMD5.equals(md5String); if (isUpdate) { // Original variable value MD5 update target.updateLastMD5(md5String); // Setting the attribute value of the @NacosValue annotation directly through the reflection mechanism if (target.method == null) { setField(target, newValue); } else { //Method callback sets property values for the @NacosValue annotation setMethod(target, newValue); } } } } } // Reflection mechanism sets Field value private void setField(final NacosValueTarget nacosValueTarget, final String propertyValue) { final Object bean = nacosValueTarget.bean; Field field = nacosValueTarget.field; String fieldName = field.getName(); try { ReflectionUtils.makeAccessible(field); field.set(bean, convertIfNecessary(field, propertyValue)); if (logger.isDebugEnabled()) { logger.debug("Update value of the {}" + " (field) in {} (bean) with {}", fieldName, nacosValueTarget.beanName, propertyValue); } } catch (Throwable e) { if (logger.isErrorEnabled()) { logger.error("Can't update value of the " + fieldName + " (field) in " + nacosValueTarget.beanName + " (bean)", e); } } } // Reflection mechanism method calls set property values private void setMethod(NacosValueTarget nacosValueTarget, String propertyValue) { Method method = nacosValueTarget.method; ReflectionUtils.makeAccessible(method); try { method.invoke(nacosValueTarget.bean, convertIfNecessary(method, propertyValue)); if (logger.isDebugEnabled()) { logger.debug("Update value with {} (method) in {} (bean) with {}", method.getName(), nacosValueTarget.beanName, propertyValue); } } catch (Throwable e) { if (logger.isErrorEnabled()) { logger.error("Can't update value with " + method.getName() + " (method) in " + nacosValueTarget.beanName + " (bean)", e); } } }
Summary: @NacosValue uses Bean-related information annotated with @NacosValue via BeanPostProcessor Cache. ConfigChangeEvent events are sent by the cache listener when configuration changes are detected through long polling. NacosValueAnnotationBeanPostProcessor listens for events, and if the attribute value MD5 matching @NacosValue is inconsistent with the newly acquired MD5 value, the Bean's @NacosValue identifying attribute is dynamically updated through the reflection mechanism to achieve hot loading