001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.pool2.impl;
018
019import java.time.Duration;
020import java.util.ArrayList;
021import java.util.Deque;
022import java.util.HashMap;
023import java.util.Iterator;
024import java.util.List;
025import java.util.Map;
026import java.util.Map.Entry;
027import java.util.NoSuchElementException;
028import java.util.Objects;
029import java.util.TreeMap;
030import java.util.concurrent.ConcurrentHashMap;
031import java.util.concurrent.TimeUnit;
032import java.util.concurrent.atomic.AtomicInteger;
033import java.util.concurrent.atomic.AtomicLong;
034import java.util.concurrent.locks.Lock;
035import java.util.concurrent.locks.ReadWriteLock;
036import java.util.concurrent.locks.ReentrantReadWriteLock;
037import java.util.stream.Collectors;
038
039import org.apache.commons.pool2.DestroyMode;
040import org.apache.commons.pool2.KeyedObjectPool;
041import org.apache.commons.pool2.KeyedPooledObjectFactory;
042import org.apache.commons.pool2.PoolUtils;
043import org.apache.commons.pool2.PooledObject;
044import org.apache.commons.pool2.PooledObjectState;
045import org.apache.commons.pool2.SwallowedExceptionListener;
046import org.apache.commons.pool2.UsageTracking;
047
048/**
049 * A configurable {@code KeyedObjectPool} implementation.
050 * <p>
051 * When coupled with the appropriate {@link KeyedPooledObjectFactory},
052 * {@code GenericKeyedObjectPool} provides robust pooling functionality for
053 * keyed objects. A {@code GenericKeyedObjectPool} can be viewed as a map
054 * of sub-pools, keyed on the (unique) key values provided to the
055 * {@link #preparePool preparePool}, {@link #addObject addObject} or
056 * {@link #borrowObject borrowObject} methods. Each time a new key value is
057 * provided to one of these methods, a sub-new pool is created under the given
058 * key to be managed by the containing {@code GenericKeyedObjectPool.}
059 * </p>
060 * <p>
061 * Note that the current implementation uses a ConcurrentHashMap which uses
062 * equals() to compare keys.
063 * This means that distinct instance keys must be distinguishable using equals.
064 * </p>
065 * <p>
066 * Optionally, one may configure the pool to examine and possibly evict objects
067 * as they sit idle in the pool and to ensure that a minimum number of idle
068 * objects is maintained for each key. This is performed by an "idle object
069 * eviction" thread, which runs asynchronously. Caution should be used when
070 * configuring this optional feature. Eviction runs contend with client threads
071 * for access to objects in the pool, so if they run too frequently performance
072 * issues may result.
073 * </p>
074 * <p>
075 * Implementation note: To prevent possible deadlocks, care has been taken to
076 * ensure that no call to a factory method will occur within a synchronization
077 * block. See POOL-125 and DBCP-44 for more information.
078 * </p>
079 * <p>
080 * This class is intended to be thread-safe.
081 * </p>
082 *
083 * @see GenericObjectPool
084 *
085 * @param <K> The type of keys maintained by this pool.
086 * @param <T> Type of element pooled in this pool.
087 *
088 * @since 2.0
089 */
090public class GenericKeyedObjectPool<K, T> extends BaseGenericObjectPool<T>
091        implements KeyedObjectPool<K, T>, GenericKeyedObjectPoolMXBean<K>, UsageTracking<T> {
092
093    private static final Integer ZERO = Integer.valueOf(0);
094
095    /**
096     * Maintains information on the per key queue for a given key.
097     *
098     * @param <S> type of objects in the pool
099     */
100    private class ObjectDeque<S> {
101
102        private final LinkedBlockingDeque<PooledObject<S>> idleObjects;
103
104        /*
105         * Number of instances created - number destroyed.
106         * Invariant: createCount <= maxTotalPerKey
107         */
108        private final AtomicInteger createCount = new AtomicInteger(0);
109
110        private long makeObjectCount;
111        private final Object makeObjectCountLock = new Object();
112
113        /*
114         * The map is keyed on pooled instances, wrapped to ensure that
115         * they work properly as keys.
116         */
117        private final Map<IdentityWrapper<S>, PooledObject<S>> allObjects =
118                new ConcurrentHashMap<>();
119
120        /*
121         * Number of threads with registered interest in this key.
122         * register(K) increments this counter and deRegister(K) decrements it.
123         * Invariant: empty keyed pool will not be dropped unless numInterested
124         *            is 0.
125         */
126        private final AtomicLong numInterested = new AtomicLong();
127
128        /**
129         * Constructs a new ObjecDeque with the given fairness policy.
130         * @param fairness true means client threads waiting to borrow / return instances
131         * will be served as if waiting in a FIFO queue.
132         */
133        public ObjectDeque(final boolean fairness) {
134            idleObjects = new LinkedBlockingDeque<>(fairness);
135        }
136
137        /**
138         * Gets all the objects for the current key.
139         *
140         * @return All the objects
141         */
142        public Map<IdentityWrapper<S>, PooledObject<S>> getAllObjects() {
143            return allObjects;
144        }
145
146        /**
147         * Gets the count of the number of objects created for the current
148         * key.
149         *
150         * @return The number of objects created for this key
151         */
152        public AtomicInteger getCreateCount() {
153            return createCount;
154        }
155
156        /**
157         * Gets the idle objects for the current key.
158         *
159         * @return The idle objects
160         */
161        public LinkedBlockingDeque<PooledObject<S>> getIdleObjects() {
162            return idleObjects;
163        }
164
165        /**
166         * Gets the number of threads with an interest registered in this key.
167         *
168         * @return The number of threads with a registered interest in this key
169         */
170        public AtomicLong getNumInterested() {
171            return numInterested;
172        }
173
174        @Override
175        public String toString() {
176            final StringBuilder builder = new StringBuilder();
177            builder.append("ObjectDeque [idleObjects=");
178            builder.append(idleObjects);
179            builder.append(", createCount=");
180            builder.append(createCount);
181            builder.append(", allObjects=");
182            builder.append(allObjects);
183            builder.append(", numInterested=");
184            builder.append(numInterested);
185            builder.append("]");
186            return builder.toString();
187        }
188
189    }
190
191    // JMX specific attributes
192    private static final String ONAME_BASE =
193            "org.apache.commons.pool2:type=GenericKeyedObjectPool,name=";
194
195    //--- configuration attributes ---------------------------------------------
196    private volatile int maxIdlePerKey =
197            GenericKeyedObjectPoolConfig.DEFAULT_MAX_IDLE_PER_KEY;
198
199    private volatile int minIdlePerKey =
200            GenericKeyedObjectPoolConfig.DEFAULT_MIN_IDLE_PER_KEY;
201
202
203    private volatile int maxTotalPerKey =
204            GenericKeyedObjectPoolConfig.DEFAULT_MAX_TOTAL_PER_KEY;
205
206    private final KeyedPooledObjectFactory<K, T> factory;
207
208    private final boolean fairness;
209
210    /*
211     * My hash of sub-pools (ObjectQueue). The list of keys <b>must</b> be kept
212     * in step with {@link #poolKeyList} using {@link #keyLock} to ensure any
213     * changes to the list of current keys is made in a thread-safe manner.
214     */
215    private final Map<K, ObjectDeque<T>> poolMap =
216            new ConcurrentHashMap<>(); // @GuardedBy("keyLock") for write access (and some read access)
217
218    /*
219     * List of pool keys - used to control eviction order. The list of keys
220     * <b>must</b> be kept in step with {@link #poolMap} using {@link #keyLock}
221     * to ensure any changes to the list of current keys is made in a
222     * thread-safe manner.
223     */
224    private final List<K> poolKeyList = new ArrayList<>(); // @GuardedBy("keyLock")
225
226    private final ReadWriteLock keyLock = new ReentrantReadWriteLock(true);
227
228    /*
229     * The combined count of the currently active objects for all keys and those
230     * in the process of being created. Under load, it may exceed
231     * {@link #maxTotal} but there will never be more than {@link #maxTotal}
232     * created at any one time.
233     */
234    private final AtomicInteger numTotal = new AtomicInteger(0);
235
236    private Iterator<K> evictionKeyIterator; // @GuardedBy("evictionLock")
237
238
239    private K evictionKey; // @GuardedBy("evictionLock")
240
241    /**
242     * Constructs a new {@code GenericKeyedObjectPool} using defaults from
243     * {@link GenericKeyedObjectPoolConfig}.
244     * @param factory the factory to be used to create entries
245     */
246    public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory) {
247        this(factory, new GenericKeyedObjectPoolConfig<>());
248    }
249
250    /**
251     * Constructs a new {@code GenericKeyedObjectPool} using a specific
252     * configuration.
253     *
254     * @param factory the factory to be used to create entries
255     * @param config    The configuration to use for this pool instance. The
256     *                  configuration is used by value. Subsequent changes to
257     *                  the configuration object will not be reflected in the
258     *                  pool.
259     */
260    public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory,
261            final GenericKeyedObjectPoolConfig<T> config) {
262
263        super(config, ONAME_BASE, config.getJmxNamePrefix());
264
265        if (factory == null) {
266            jmxUnregister(); // tidy up
267            throw new IllegalArgumentException("Factory may not be null");
268        }
269        this.factory = factory;
270        this.fairness = config.getFairness();
271
272        setConfig(config);
273    }
274
275    /**
276     * Creates a new {@code GenericKeyedObjectPool} that tracks and destroys
277     * objects that are checked out, but never returned to the pool.
278     *
279     * @param factory   The object factory to be used to create object instances
280     *                  used by this pool
281     * @param config    The base pool configuration to use for this pool instance.
282     *                  The configuration is used by value. Subsequent changes to
283     *                  the configuration object will not be reflected in the
284     *                  pool.
285     * @param abandonedConfig  Configuration for abandoned object identification
286     *                         and removal.  The configuration is used by value.
287     * @since 2.10.0
288     */
289    public GenericKeyedObjectPool(final KeyedPooledObjectFactory<K, T> factory,
290            final GenericKeyedObjectPoolConfig<T> config, final AbandonedConfig abandonedConfig) {
291        this(factory, config);
292        setAbandonedConfig(abandonedConfig);
293    }
294
295    /**
296     * Add an object to the set of idle objects for a given key.
297     *
298     * @param key The key to associate with the idle object
299     * @param p The wrapped object to add.
300     *
301     * @throws Exception If the associated factory fails to passivate the object
302     */
303    private void addIdleObject(final K key, final PooledObject<T> p) throws Exception {
304
305        if (p != null) {
306            factory.passivateObject(key, p);
307            final LinkedBlockingDeque<PooledObject<T>> idleObjects =
308                    poolMap.get(key).getIdleObjects();
309            if (getLifo()) {
310                idleObjects.addFirst(p);
311            } else {
312                idleObjects.addLast(p);
313            }
314        }
315    }
316
317
318    /**
319     * Create an object using the {@link KeyedPooledObjectFactory#makeObject
320     * factory}, passivate it, and then place it in the idle object pool.
321     * {@code addObject} is useful for "pre-loading" a pool with idle
322     * objects.
323     *
324     * @param key the key a new instance should be added to
325     *
326     * @throws Exception when {@link KeyedPooledObjectFactory#makeObject}
327     *                   fails.
328     */
329    @Override
330    public void addObject(final K key) throws Exception {
331        assertOpen();
332        register(key);
333        try {
334            final PooledObject<T> p = create(key);
335            addIdleObject(key, p);
336        } finally {
337            deregister(key);
338        }
339    }
340
341
342    /**
343     * Equivalent to <code>{@link #borrowObject(Object, long) borrowObject}(key,
344     * {@link #getMaxWaitDuration()})</code>.
345     *
346     * {@inheritDoc}
347     */
348    @Override
349    public T borrowObject(final K key) throws Exception {
350        return borrowObject(key, getMaxWaitDuration().toMillis());
351    }
352
353
354    /**
355     * Borrows an object from the sub-pool associated with the given key using
356     * the specified waiting time which only applies if
357     * {@link #getBlockWhenExhausted()} is true.
358     * <p>
359     * If there is one or more idle instances available in the sub-pool
360     * associated with the given key, then an idle instance will be selected
361     * based on the value of {@link #getLifo()}, activated and returned.  If
362     * activation fails, or {@link #getTestOnBorrow() testOnBorrow} is set to
363     * {@code true} and validation fails, the instance is destroyed and the
364     * next available instance is examined.  This continues until either a valid
365     * instance is returned or there are no more idle instances available.
366     * </p>
367     * <p>
368     * If there are no idle instances available in the sub-pool associated with
369     * the given key, behavior depends on the {@link #getMaxTotalPerKey()
370     * maxTotalPerKey}, {@link #getMaxTotal() maxTotal}, and (if applicable)
371     * {@link #getBlockWhenExhausted()} and the value passed in to the
372     * {@code borrowMaxWaitMillis} parameter. If the number of instances checked
373     * out from the sub-pool under the given key is less than
374     * {@code maxTotalPerKey} and the total number of instances in
375     * circulation (under all keys) is less than {@code maxTotal}, a new
376     * instance is created, activated and (if applicable) validated and returned
377     * to the caller. If validation fails, a {@code NoSuchElementException}
378     * will be thrown.
379     * </p>
380     * <p>
381     * If the associated sub-pool is exhausted (no available idle instances and
382     * no capacity to create new ones), this method will either block
383     * ({@link #getBlockWhenExhausted()} is true) or throw a
384     * {@code NoSuchElementException}
385     * ({@link #getBlockWhenExhausted()} is false).
386     * The length of time that this method will block when
387     * {@link #getBlockWhenExhausted()} is true is determined by the value
388     * passed in to the {@code borrowMaxWait} parameter.
389     * </p>
390     * <p>
391     * When {@code maxTotal} is set to a positive value and this method is
392     * invoked when at the limit with no idle instances available under the requested
393     * key, an attempt is made to create room by clearing the oldest 15% of the
394     * elements from the keyed sub-pools.
395     * </p>
396     * <p>
397     * When the pool is exhausted, multiple calling threads may be
398     * simultaneously blocked waiting for instances to become available. A
399     * "fairness" algorithm has been implemented to ensure that threads receive
400     * available instances in request arrival order.
401     * </p>
402     *
403     * @param key pool key
404     * @param borrowMaxWaitMillis The time to wait in milliseconds for an object
405     *                            to become available
406     *
407     * @return object instance from the keyed pool
408     *
409     * @throws NoSuchElementException if a keyed object instance cannot be
410     *                                returned because the pool is exhausted.
411     *
412     * @throws Exception if a keyed object instance cannot be returned due to an
413     *                   error
414     */
415    public T borrowObject(final K key, final long borrowMaxWaitMillis) throws Exception {
416        assertOpen();
417
418        final AbandonedConfig ac = this.abandonedConfig;
419        if (ac != null && ac.getRemoveAbandonedOnBorrow() && (getNumIdle() < 2) &&
420                (getNumActive() > getMaxTotal() - 3)) {
421            removeAbandoned(ac);
422        }
423
424        PooledObject<T> p = null;
425
426        // Get local copy of current config so it is consistent for entire
427        // method execution
428        final boolean blockWhenExhausted = getBlockWhenExhausted();
429
430        boolean create;
431        final long waitTimeMillis = System.currentTimeMillis();
432        final ObjectDeque<T> objectDeque = register(key);
433
434        try {
435            while (p == null) {
436                create = false;
437                p = objectDeque.getIdleObjects().pollFirst();
438                if (p == null) {
439                    p = create(key);
440                    if (p != null) {
441                        create = true;
442                    }
443                }
444                if (blockWhenExhausted) {
445                    if (p == null) {
446                        if (borrowMaxWaitMillis < 0) {
447                            p = objectDeque.getIdleObjects().takeFirst();
448                        } else {
449                            p = objectDeque.getIdleObjects().pollFirst(borrowMaxWaitMillis, TimeUnit.MILLISECONDS);
450                        }
451                    }
452                    if (p == null) {
453                        throw new NoSuchElementException(appendStats(
454                                "Timeout waiting for idle object, borrowMaxWaitMillis=" + borrowMaxWaitMillis));
455                    }
456                } else if (p == null) {
457                    throw new NoSuchElementException(appendStats("Pool exhausted"));
458                }
459                if (!p.allocate()) {
460                    p = null;
461                }
462
463                if (p != null) {
464                    try {
465                        factory.activateObject(key, p);
466                    } catch (final Exception e) {
467                        try {
468                            destroy(key, p, true, DestroyMode.NORMAL);
469                        } catch (final Exception e1) {
470                            // Ignore - activation failure is more important
471                        }
472                        p = null;
473                        if (create) {
474                            final NoSuchElementException nsee = new NoSuchElementException(appendStats("Unable to activate object"));
475                            nsee.initCause(e);
476                            throw nsee;
477                        }
478                    }
479                    if (p != null && getTestOnBorrow()) {
480                        boolean validate = false;
481                        Throwable validationThrowable = null;
482                        try {
483                            validate = factory.validateObject(key, p);
484                        } catch (final Throwable t) {
485                            PoolUtils.checkRethrow(t);
486                            validationThrowable = t;
487                        }
488                        if (!validate) {
489                            try {
490                                destroy(key, p, true, DestroyMode.NORMAL);
491                                destroyedByBorrowValidationCount.incrementAndGet();
492                            } catch (final Exception e) {
493                                // Ignore - validation failure is more important
494                            }
495                            p = null;
496                            if (create) {
497                                final NoSuchElementException nsee = new NoSuchElementException(
498                                        appendStats("Unable to validate object"));
499                                nsee.initCause(validationThrowable);
500                                throw nsee;
501                            }
502                        }
503                    }
504                }
505            }
506        } finally {
507            deregister(key);
508        }
509
510        updateStatsBorrow(p, Duration.ofMillis(System.currentTimeMillis() - waitTimeMillis));
511
512        return p.getObject();
513    }
514
515    @Override
516    String getStatsString() {
517        // Simply listed in AB order.
518        return super.getStatsString() +
519                String.format(", fairness=%s, maxIdlePerKey%,d, maxTotalPerKey=%,d, minIdlePerKey=%,d, numTotal=%,d",
520                        fairness, maxIdlePerKey, maxTotalPerKey, minIdlePerKey, numTotal.get());
521    }
522
523    /**
524     * Calculate the number of objects that need to be created to attempt to
525     * maintain the minimum number of idle objects while not exceeded the limits
526     * on the maximum number of objects either per key or totally.
527     *
528     * @param objectDeque   The set of objects to check
529     *
530     * @return The number of new objects to create
531     */
532    private int calculateDeficit(final ObjectDeque<T> objectDeque) {
533
534        if (objectDeque == null) {
535            return getMinIdlePerKey();
536        }
537
538        // Used more than once so keep a local copy so the value is consistent
539        final int maxTotal = getMaxTotal();
540        final int maxTotalPerKeySave = getMaxTotalPerKey();
541
542        // Calculate no of objects needed to be created, in order to have
543        // the number of pooled objects < maxTotalPerKey();
544        int objectDefecit = getMinIdlePerKey() - objectDeque.getIdleObjects().size();
545        if (maxTotalPerKeySave > 0) {
546            final int growLimit = Math.max(0,
547                    maxTotalPerKeySave - objectDeque.getIdleObjects().size());
548            objectDefecit = Math.min(objectDefecit, growLimit);
549        }
550
551        // Take the maxTotal limit into account
552        if (maxTotal > 0) {
553            final int growLimit = Math.max(0, maxTotal - getNumActive() - getNumIdle());
554            objectDefecit = Math.min(objectDefecit, growLimit);
555        }
556
557        return objectDefecit;
558    }
559
560
561    /**
562     * Clears any objects sitting idle in the pool by removing them from the
563     * idle instance sub-pools and then invoking the configured
564     * PoolableObjectFactory's
565     * {@link KeyedPooledObjectFactory#destroyObject(Object, PooledObject)}
566     * method on each idle instance.
567     * <p>
568     * Implementation notes:
569     * </p>
570     * <ul>
571     * <li>This method does not destroy or effect in any way instances that are
572     * checked out when it is invoked.</li>
573     * <li>Invoking this method does not prevent objects being returned to the
574     * idle instance pool, even during its execution. Additional instances may
575     * be returned while removed items are being destroyed.</li>
576     * <li>Exceptions encountered destroying idle instances are swallowed
577     * but notified via a {@link SwallowedExceptionListener}.</li>
578     * </ul>
579     */
580    @Override
581    public void clear() {
582        poolMap.keySet().forEach(this::clear);
583    }
584
585
586    /**
587     * Clears the specified sub-pool, removing all pooled instances
588     * corresponding to the given {@code key}. Exceptions encountered
589     * destroying idle instances are swallowed but notified via a
590     * {@link SwallowedExceptionListener}.
591     *
592     * @param key the key to clear
593     */
594    @Override
595    public void clear(final K key) {
596
597        final ObjectDeque<T> objectDeque = register(key);
598
599        try {
600            final LinkedBlockingDeque<PooledObject<T>> idleObjects =
601                    objectDeque.getIdleObjects();
602
603            PooledObject<T> p = idleObjects.poll();
604
605            while (p != null) {
606                try {
607                    destroy(key, p, true, DestroyMode.NORMAL);
608                } catch (final Exception e) {
609                    swallowException(e);
610                }
611                p = idleObjects.poll();
612            }
613        } finally {
614            deregister(key);
615        }
616    }
617
618
619    /**
620     * Clears oldest 15% of objects in pool.  The method sorts the objects into
621     * a TreeMap and then iterates the first 15% for removal.
622     */
623    public void clearOldest() {
624
625        // build sorted map of idle objects
626        final Map<PooledObject<T>, K> map = new TreeMap<>();
627
628        poolMap.forEach((key, value) -> {
629            // Protect against possible NPE if key has been removed in another
630            // thread. Not worth locking the keys while this loop completes.
631            if (value != null) {
632                // Each item into the map using the PooledObject object as the
633                // key. It then gets sorted based on the idle time
634                value.getIdleObjects().forEach(p -> map.put(p, key));
635            }
636        });
637
638        // Now iterate created map and kill the first 15% plus one to account
639        // for zero
640        int itemsToRemove = ((int) (map.size() * 0.15)) + 1;
641        final Iterator<Entry<PooledObject<T>, K>> iter = map.entrySet().iterator();
642
643        while (iter.hasNext() && itemsToRemove > 0) {
644            final Entry<PooledObject<T>, K> entry = iter.next();
645            // kind of backwards on naming.  In the map, each key is the
646            // PooledObject because it has the ordering with the timestamp
647            // value.  Each value that the key references is the key of the
648            // list it belongs to.
649            final K key = entry.getValue();
650            final PooledObject<T> p = entry.getKey();
651            // Assume the destruction succeeds
652            boolean destroyed = true;
653            try {
654                destroyed = destroy(key, p, false, DestroyMode.NORMAL);
655            } catch (final Exception e) {
656                swallowException(e);
657            }
658            if (destroyed) {
659                itemsToRemove--;
660            }
661        }
662    }
663
664
665    /**
666     * Closes the keyed object pool. Once the pool is closed,
667     * {@link #borrowObject(Object)} will fail with IllegalStateException, but
668     * {@link #returnObject(Object, Object)} and
669     * {@link #invalidateObject(Object, Object)} will continue to work, with
670     * returned objects destroyed on return.
671     * <p>
672     * Destroys idle instances in the pool by invoking {@link #clear()}.
673     * </p>
674     */
675    @Override
676    public void close() {
677        if (isClosed()) {
678            return;
679        }
680
681        synchronized (closeLock) {
682            if (isClosed()) {
683                return;
684            }
685
686            // Stop the evictor before the pool is closed since evict() calls
687            // assertOpen()
688            stopEvictor();
689
690            closed = true;
691            // This clear removes any idle objects
692            clear();
693
694            jmxUnregister();
695
696            // Release any threads that were waiting for an object
697            poolMap.values().forEach(e -> e.getIdleObjects().interuptTakeWaiters());
698            // This clear cleans up the keys now any waiting threads have been
699            // interrupted
700            clear();
701        }
702    }
703
704    /**
705     * Creates a new pooled object.
706     *
707     * @param key Key associated with new pooled object
708     *
709     * @return The new, wrapped pooled object
710     *
711     * @throws Exception If the objection creation fails
712     */
713    private PooledObject<T> create(final K key) throws Exception {
714        int maxTotalPerKeySave = getMaxTotalPerKey(); // Per key
715        if (maxTotalPerKeySave < 0) {
716            maxTotalPerKeySave = Integer.MAX_VALUE;
717        }
718        final int maxTotal = getMaxTotal();   // All keys
719
720        final ObjectDeque<T> objectDeque = poolMap.get(key);
721
722        // Check against the overall limit
723        boolean loop = true;
724
725        while (loop) {
726            final int newNumTotal = numTotal.incrementAndGet();
727            if (maxTotal > -1 && newNumTotal > maxTotal) {
728                numTotal.decrementAndGet();
729                if (getNumIdle() == 0) {
730                    return null;
731                }
732                clearOldest();
733            } else {
734                loop = false;
735            }
736        }
737
738        // Flag that indicates if create should:
739        // - TRUE:  call the factory to create an object
740        // - FALSE: return null
741        // - null:  loop and re-test the condition that determines whether to
742        //          call the factory
743        Boolean create = null;
744        while (create == null) {
745            synchronized (objectDeque.makeObjectCountLock) {
746                final long newCreateCount = objectDeque.getCreateCount().incrementAndGet();
747                // Check against the per key limit
748                if (newCreateCount > maxTotalPerKeySave) {
749                    // The key is currently at capacity or in the process of
750                    // making enough new objects to take it to capacity.
751                    objectDeque.getCreateCount().decrementAndGet();
752                    if (objectDeque.makeObjectCount == 0) {
753                        // There are no makeObject() calls in progress for this
754                        // key so the key is at capacity. Do not attempt to
755                        // create a new object. Return and wait for an object to
756                        // be returned.
757                        create = Boolean.FALSE;
758                    } else {
759                        // There are makeObject() calls in progress that might
760                        // bring the pool to capacity. Those calls might also
761                        // fail so wait until they complete and then re-test if
762                        // the pool is at capacity or not.
763                        objectDeque.makeObjectCountLock.wait();
764                    }
765                } else {
766                    // The pool is not at capacity. Create a new object.
767                    objectDeque.makeObjectCount++;
768                    create = Boolean.TRUE;
769                }
770            }
771        }
772
773        if (!create.booleanValue()) {
774            numTotal.decrementAndGet();
775            return null;
776        }
777
778        PooledObject<T> p = null;
779        try {
780            p = factory.makeObject(key);
781            if (getTestOnCreate() && !factory.validateObject(key, p)) {
782                numTotal.decrementAndGet();
783                objectDeque.getCreateCount().decrementAndGet();
784                return null;
785            }
786        } catch (final Exception e) {
787            numTotal.decrementAndGet();
788            objectDeque.getCreateCount().decrementAndGet();
789            throw e;
790        } finally {
791            synchronized (objectDeque.makeObjectCountLock) {
792                objectDeque.makeObjectCount--;
793                objectDeque.makeObjectCountLock.notifyAll();
794            }
795        }
796
797        final AbandonedConfig ac = this.abandonedConfig;
798        if (ac != null && ac.getLogAbandoned()) {
799            p.setLogAbandoned(true);
800            p.setRequireFullStackTrace(ac.getRequireFullStackTrace());
801        }
802
803        createdCount.incrementAndGet();
804        objectDeque.getAllObjects().put(new IdentityWrapper<>(p.getObject()), p);
805        return p;
806    }
807
808    /**
809     * De-register the use of a key by an object.
810     * <p>
811     * {@link #register(Object)} and {@link #deregister(Object)} must always be used as a pair.
812     * </p>
813     *
814     * @param k The key to de-register
815     */
816    private void deregister(final K k) {
817        Lock lock = keyLock.readLock();
818        try {
819            lock.lock();
820            final ObjectDeque<T> objectDeque = poolMap.get(k);
821            final long numInterested = objectDeque.getNumInterested().decrementAndGet();
822            if (numInterested == 0 && objectDeque.getCreateCount().get() == 0) {
823                // Potential to remove key
824                // Upgrade to write lock
825                lock.unlock();
826                lock = keyLock.writeLock();
827                lock.lock();
828                if (objectDeque.getCreateCount().get() == 0 && objectDeque.getNumInterested().get() == 0) {
829                    // NOTE: Keys must always be removed from both poolMap and
830                    //       poolKeyList at the same time while protected by
831                    //       keyLock.writeLock()
832                    poolMap.remove(k);
833                    poolKeyList.remove(k);
834                }
835            }
836        } finally {
837            lock.unlock();
838        }
839    }
840
841
842    /**
843     * Destroy the wrapped, pooled object.
844     *
845     * @param key The key associated with the object to destroy.
846     * @param toDestroy The wrapped object to be destroyed
847     * @param always Should the object be destroyed even if it is not currently
848     *               in the set of idle objects for the given key
849     * @param destroyMode DestroyMode context provided to the factory
850     *
851     * @return {@code true} if the object was destroyed, otherwise {@code false}
852     * @throws Exception If the object destruction failed
853     */
854    private boolean destroy(final K key, final PooledObject<T> toDestroy, final boolean always, final DestroyMode destroyMode)
855            throws Exception {
856
857        final ObjectDeque<T> objectDeque = register(key);
858
859        try {
860            boolean isIdle;
861            synchronized(toDestroy) {
862                // Check idle state directly
863                isIdle = toDestroy.getState().equals(PooledObjectState.IDLE);
864                // If idle, not under eviction test, or always is true, remove instance,
865                // updating isIdle if instance is found in idle objects
866                if (isIdle || always) {
867                    isIdle = objectDeque.getIdleObjects().remove(toDestroy);
868                }
869            }
870            if (isIdle || always) {
871                objectDeque.getAllObjects().remove(new IdentityWrapper<>(toDestroy.getObject()));
872                toDestroy.invalidate();
873
874                try {
875                    factory.destroyObject(key, toDestroy, destroyMode);
876                } finally {
877                    objectDeque.getCreateCount().decrementAndGet();
878                    destroyedCount.incrementAndGet();
879                    numTotal.decrementAndGet();
880                }
881                return true;
882            }
883            return false;
884        } finally {
885            deregister(key);
886        }
887    }
888
889    @Override
890    void ensureMinIdle() throws Exception {
891        final int minIdlePerKeySave = getMinIdlePerKey();
892        if (minIdlePerKeySave < 1) {
893            return;
894        }
895
896        for (final K k : poolMap.keySet()) {
897            ensureMinIdle(k);
898        }
899    }
900
901    /**
902     * Ensure that the configured number of minimum idle objects is available in
903     * the pool for the given key.
904     *
905     * @param key The key to check for idle objects
906     *
907     * @throws Exception If a new object is required and cannot be created
908     */
909    private void ensureMinIdle(final K key) throws Exception {
910        // Calculate current pool objects
911        ObjectDeque<T> objectDeque = poolMap.get(key);
912
913        // objectDeque == null is OK here. It is handled correctly by both
914        // methods called below.
915
916        // this method isn't synchronized so the
917        // calculateDeficit is done at the beginning
918        // as a loop limit and a second time inside the loop
919        // to stop when another thread already returned the
920        // needed objects
921        final int deficit = calculateDeficit(objectDeque);
922
923        for (int i = 0; i < deficit && calculateDeficit(objectDeque) > 0; i++) {
924            addObject(key);
925            // If objectDeque was null, it won't be any more. Obtain a reference
926            // to it so the deficit can be correctly calculated. It needs to
927            // take account of objects created in other threads.
928            if (objectDeque == null) {
929                objectDeque = poolMap.get(key);
930            }
931        }
932    }
933
934
935    /**
936     * {@inheritDoc}
937     * <p>
938     * Successive activations of this method examine objects in keyed sub-pools
939     * in sequence, cycling through the keys and examining objects in
940     * oldest-to-youngest order within the keyed sub-pools.
941     * </p>
942     */
943    @Override
944    public void evict() throws Exception {
945        assertOpen();
946
947        if (getNumIdle() > 0) {
948
949            PooledObject<T> underTest = null;
950            final EvictionPolicy<T> evictionPolicy = getEvictionPolicy();
951
952            synchronized (evictionLock) {
953                final EvictionConfig evictionConfig = new EvictionConfig(
954                        getMinEvictableIdleDuration(),
955                        getSoftMinEvictableIdleDuration(),
956                        getMinIdlePerKey());
957
958                final boolean testWhileIdle = getTestWhileIdle();
959
960                for (int i = 0, m = getNumTests(); i < m; i++) {
961                    if (evictionIterator == null || !evictionIterator.hasNext()) {
962                        if (evictionKeyIterator == null ||
963                                !evictionKeyIterator.hasNext()) {
964                            final List<K> keyCopy = new ArrayList<>();
965                            final Lock readLock = keyLock.readLock();
966                            readLock.lock();
967                            try {
968                                keyCopy.addAll(poolKeyList);
969                            } finally {
970                                readLock.unlock();
971                            }
972                            evictionKeyIterator = keyCopy.iterator();
973                        }
974                        while (evictionKeyIterator.hasNext()) {
975                            evictionKey = evictionKeyIterator.next();
976                            final ObjectDeque<T> objectDeque = poolMap.get(evictionKey);
977                            if (objectDeque == null) {
978                                continue;
979                            }
980
981                            final Deque<PooledObject<T>> idleObjects = objectDeque.getIdleObjects();
982                            evictionIterator = new EvictionIterator(idleObjects);
983                            if (evictionIterator.hasNext()) {
984                                break;
985                            }
986                            evictionIterator = null;
987                        }
988                    }
989                    if (evictionIterator == null) {
990                        // Pools exhausted
991                        return;
992                    }
993                    final Deque<PooledObject<T>> idleObjects;
994                    try {
995                        underTest = evictionIterator.next();
996                        idleObjects = evictionIterator.getIdleObjects();
997                    } catch (final NoSuchElementException nsee) {
998                        // Object was borrowed in another thread
999                        // Don't count this as an eviction test so reduce i;
1000                        i--;
1001                        evictionIterator = null;
1002                        continue;
1003                    }
1004
1005                    if (!underTest.startEvictionTest()) {
1006                        // Object was borrowed in another thread
1007                        // Don't count this as an eviction test so reduce i;
1008                        i--;
1009                        continue;
1010                    }
1011
1012                    // User provided eviction policy could throw all sorts of
1013                    // crazy exceptions. Protect against such an exception
1014                    // killing the eviction thread.
1015                    boolean evict;
1016                    try {
1017                        evict = evictionPolicy.evict(evictionConfig, underTest,
1018                                poolMap.get(evictionKey).getIdleObjects().size());
1019                    } catch (final Throwable t) {
1020                        // Slightly convoluted as SwallowedExceptionListener
1021                        // uses Exception rather than Throwable
1022                        PoolUtils.checkRethrow(t);
1023                        swallowException(new Exception(t));
1024                        // Don't evict on error conditions
1025                        evict = false;
1026                    }
1027
1028                    if (evict) {
1029                        destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1030                        destroyedByEvictorCount.incrementAndGet();
1031                    } else {
1032                        if (testWhileIdle) {
1033                            boolean active = false;
1034                            try {
1035                                factory.activateObject(evictionKey, underTest);
1036                                active = true;
1037                            } catch (final Exception e) {
1038                                destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1039                                destroyedByEvictorCount.incrementAndGet();
1040                            }
1041                            if (active) {
1042                                boolean validate = false;
1043                                Throwable validationThrowable = null;
1044                                try {
1045                                    validate = factory.validateObject(evictionKey, underTest);
1046                                } catch (final Throwable t) {
1047                                    PoolUtils.checkRethrow(t);
1048                                    validationThrowable = t;
1049                                }
1050                                if (!validate) {
1051                                    destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1052                                    destroyedByEvictorCount.incrementAndGet();
1053                                    if (validationThrowable != null) {
1054                                        if (validationThrowable instanceof RuntimeException) {
1055                                            throw (RuntimeException) validationThrowable;
1056                                        }
1057                                        throw (Error) validationThrowable;
1058                                    }
1059                                } else {
1060                                    try {
1061                                        factory.passivateObject(evictionKey, underTest);
1062                                    } catch (final Exception e) {
1063                                        destroy(evictionKey, underTest, true, DestroyMode.NORMAL);
1064                                        destroyedByEvictorCount.incrementAndGet();
1065                                    }
1066                                }
1067                            }
1068                        }
1069                        if (!underTest.endEvictionTest(idleObjects)) {
1070                            // TODO - May need to add code here once additional
1071                            // states are used
1072                        }
1073                    }
1074                }
1075            }
1076        }
1077        final AbandonedConfig ac = this.abandonedConfig;
1078        if (ac != null && ac.getRemoveAbandonedOnMaintenance()) {
1079            removeAbandoned(ac);
1080        }
1081    }
1082
1083    /**
1084     * Gets a reference to the factory used to create, destroy and validate
1085     * the objects used by this pool.
1086     *
1087     * @return the factory
1088     */
1089    public KeyedPooledObjectFactory<K, T> getFactory() {
1090        return factory;
1091    }
1092
1093    /**
1094     * Gets the cap on the number of "idle" instances per key in the pool.
1095     * If maxIdlePerKey is set too low on heavily loaded systems it is possible
1096     * you will see objects being destroyed and almost immediately new objects
1097     * being created. This is a result of the active threads momentarily
1098     * returning objects faster than they are requesting them, causing the
1099     * number of idle objects to rise above maxIdlePerKey. The best value for
1100     * maxIdlePerKey for heavily loaded system will vary but the default is a
1101     * good starting point.
1102     *
1103     * @return the maximum number of "idle" instances that can be held in a
1104     *         given keyed sub-pool or a negative value if there is no limit
1105     *
1106     * @see #setMaxIdlePerKey
1107     */
1108    @Override
1109    public int getMaxIdlePerKey() {
1110        return maxIdlePerKey;
1111    }
1112
1113    /**
1114     * Gets the limit on the number of object instances allocated by the pool
1115     * (checked out or idle), per key. When the limit is reached, the sub-pool
1116     * is said to be exhausted. A negative value indicates no limit.
1117     *
1118     * @return the limit on the number of active instances per key
1119     *
1120     * @see #setMaxTotalPerKey
1121     */
1122    @Override
1123    public int getMaxTotalPerKey() {
1124        return maxTotalPerKey;
1125    }
1126
1127    /**
1128     * Gets the target for the minimum number of idle objects to maintain in
1129     * each of the keyed sub-pools. This setting only has an effect if it is
1130     * positive and {@link #getDurationBetweenEvictionRuns()} is greater than
1131     * zero. If this is the case, an attempt is made to ensure that each
1132     * sub-pool has the required minimum number of instances during idle object
1133     * eviction runs.
1134     * <p>
1135     * If the configured value of minIdlePerKey is greater than the configured
1136     * value for maxIdlePerKey then the value of maxIdlePerKey will be used
1137     * instead.
1138     * </p>
1139     *
1140     * @return minimum size of the each keyed pool
1141     *
1142     * @see #setTimeBetweenEvictionRunsMillis
1143     */
1144    @Override
1145    public int getMinIdlePerKey() {
1146        final int maxIdlePerKeySave = getMaxIdlePerKey();
1147        return this.minIdlePerKey > maxIdlePerKeySave ? maxIdlePerKeySave : minIdlePerKey;
1148    }
1149
1150    @Override
1151    public int getNumActive() {
1152        return numTotal.get() - getNumIdle();
1153    }
1154
1155    @Override
1156    public int getNumActive(final K key) {
1157        final ObjectDeque<T> objectDeque = poolMap.get(key);
1158        if (objectDeque != null) {
1159            return objectDeque.getAllObjects().size() -
1160                    objectDeque.getIdleObjects().size();
1161        }
1162        return 0;
1163    }
1164
1165    @Override
1166    public Map<String, Integer> getNumActivePerKey() {
1167        final HashMap<String, Integer> result = new HashMap<>();
1168
1169        poolMap.entrySet().forEach(entry -> {
1170            if (entry != null) {
1171                final K key = entry.getKey();
1172                final ObjectDeque<T> objectDequeue = entry.getValue();
1173                if (key != null && objectDequeue != null) {
1174                    result.put(key.toString(), Integer.valueOf(
1175                            objectDequeue.getAllObjects().size() -
1176                            objectDequeue.getIdleObjects().size()));
1177                }
1178            }
1179        });
1180        return result;
1181    }
1182
1183    @Override
1184    public int getNumIdle() {
1185        return poolMap.values().stream().mapToInt(e -> e.getIdleObjects().size()).sum();
1186    }
1187
1188
1189    //--- JMX support ----------------------------------------------------------
1190
1191    @Override
1192    public int getNumIdle(final K key) {
1193        final ObjectDeque<T> objectDeque = poolMap.get(key);
1194        return objectDeque != null ? objectDeque.getIdleObjects().size() : 0;
1195    }
1196
1197    /**
1198     * Calculate the number of objects to test in a run of the idle object
1199     * evictor.
1200     *
1201     * @return The number of objects to test for validity
1202     */
1203    private int getNumTests() {
1204        final int totalIdle = getNumIdle();
1205        final int numTests = getNumTestsPerEvictionRun();
1206        if (numTests >= 0) {
1207            return Math.min(numTests, totalIdle);
1208        }
1209        return (int) (Math.ceil(totalIdle / Math.abs((double) numTests)));
1210    }
1211
1212    /**
1213     * Return an estimate of the number of threads currently blocked waiting for
1214     * an object from the pool. This is intended for monitoring only, not for
1215     * synchronization control.
1216     *
1217     * @return The estimate of the number of threads currently blocked waiting
1218     *         for an object from the pool
1219     */
1220    @Override
1221    public int getNumWaiters() {
1222        if (getBlockWhenExhausted()) {
1223            // Assume no overflow
1224            return poolMap.values().stream().mapToInt(e -> e.getIdleObjects().getTakeQueueLength()).sum();
1225        }
1226        return 0;
1227    }
1228
1229    /**
1230     * Return an estimate of the number of threads currently blocked waiting for
1231     * an object from the pool for each key. This is intended for
1232     * monitoring only, not for synchronization control.
1233     *
1234     * @return The estimate of the number of threads currently blocked waiting
1235     *         for an object from the pool for each key
1236     */
1237    @Override
1238    public Map<String, Integer> getNumWaitersByKey() {
1239        final Map<String, Integer> result = new HashMap<>();
1240
1241        for (final Entry<K, ObjectDeque<T>> entry : poolMap.entrySet()) {
1242            final K k = entry.getKey();
1243            final ObjectDeque<T> deque = entry.getValue();
1244            if (deque != null) {
1245                result.put(k.toString(), getBlockWhenExhausted() ? Integer.valueOf(deque.getIdleObjects().getTakeQueueLength()) : ZERO);
1246            }
1247        }
1248        return result;
1249    }
1250
1251    /**
1252     * Checks to see if there are any threads currently waiting to borrow
1253     * objects but are blocked waiting for more objects to become available.
1254     *
1255     * @return {@code true} if there is at least one thread waiting otherwise
1256     *         {@code false}
1257     */
1258    private boolean hasBorrowWaiters() {
1259        return poolMap.values().stream().anyMatch(deque -> deque != null && deque.getIdleObjects().hasTakeWaiters());
1260    }
1261
1262    /**
1263     * {@inheritDoc}
1264     * <p>
1265     * Activation of this method decrements the active count associated with
1266     * the given keyed pool and attempts to destroy {@code obj.}
1267     * </p>
1268     *
1269     * @param key pool key
1270     * @param obj instance to invalidate
1271     *
1272     * @throws Exception             if an exception occurs destroying the
1273     *                               object
1274     * @throws IllegalStateException if obj does not belong to the pool
1275     *                               under the given key
1276     */
1277    @Override
1278    public void invalidateObject(final K key, final T obj) throws Exception {
1279        invalidateObject(key, obj, DestroyMode.NORMAL);
1280    }
1281
1282    /**
1283     * {@inheritDoc}
1284     * <p>
1285     * Activation of this method decrements the active count associated with
1286     * the given keyed pool and attempts to destroy {@code obj.}
1287     * </p>
1288     *
1289     * @param key pool key
1290     * @param obj instance to invalidate
1291     * @param destroyMode DestroyMode context provided to factory
1292     *
1293     * @throws Exception             if an exception occurs destroying the
1294     *                               object
1295     * @throws IllegalStateException if obj does not belong to the pool
1296     *                               under the given key
1297     * @since 2.9.0
1298     */
1299    @Override
1300    public void invalidateObject(final K key, final T obj, final DestroyMode destroyMode) throws Exception {
1301        final ObjectDeque<T> objectDeque = poolMap.get(key);
1302        final PooledObject<T> p = objectDeque.getAllObjects().get(new IdentityWrapper<>(obj));
1303        if (p == null) {
1304            throw new IllegalStateException(appendStats("Object not currently part of this pool"));
1305        }
1306        synchronized (p) {
1307            if (p.getState() != PooledObjectState.INVALID) {
1308                destroy(key, p, true, destroyMode);
1309            }
1310        }
1311        if (objectDeque.idleObjects.hasTakeWaiters()) {
1312            addObject(key);
1313        }
1314    }
1315
1316    /**
1317     * Provides information on all the objects in the pool, both idle (waiting
1318     * to be borrowed) and active (currently borrowed).
1319     * <p>
1320     * Note: This is named listAllObjects so it is presented as an operation via
1321     * JMX. That means it won't be invoked unless the explicitly requested
1322     * whereas all attributes will be automatically requested when viewing the
1323     * attributes for an object in a tool like JConsole.
1324     * </p>
1325     *
1326     * @return Information grouped by key on all the objects in the pool
1327     */
1328    @Override
1329    public Map<String, List<DefaultPooledObjectInfo>> listAllObjects() {
1330        final Map<String, List<DefaultPooledObjectInfo>> result = new HashMap<>();
1331
1332        poolMap.forEach((k, value) -> {
1333            if (value != null) {
1334                result.put(k.toString(), value.getAllObjects().values().stream().map(DefaultPooledObjectInfo::new).collect(Collectors.toList()));
1335            }
1336        });
1337        return result;
1338    }
1339
1340    /**
1341     * Registers a key for pool control and ensures that
1342     * {@link #getMinIdlePerKey()} idle instances are created.
1343     *
1344     * @param key - The key to register for pool control.
1345     *
1346     * @throws Exception If the associated factory throws an exception
1347     */
1348    public void preparePool(final K key) throws Exception {
1349        final int minIdlePerKeySave = getMinIdlePerKey();
1350        if (minIdlePerKeySave < 1) {
1351            return;
1352        }
1353        ensureMinIdle(key);
1354    }
1355
1356    /**
1357     * Register the use of a key by an object.
1358     * <p>
1359     * {@link #register(Object)} and {@link #deregister(Object)} must always be used as a pair.
1360     * </p>
1361     *
1362     * @param k The key to register
1363     *
1364     * @return The objects currently associated with the given key. If this
1365     *         method returns without throwing an exception then it will never
1366     *         return null.
1367     */
1368    private ObjectDeque<T> register(final K k) {
1369        Lock lock = keyLock.readLock();
1370        ObjectDeque<T> objectDeque = null;
1371        try {
1372            lock.lock();
1373            objectDeque = poolMap.get(k);
1374            if (objectDeque == null) {
1375                // Upgrade to write lock
1376                lock.unlock();
1377                lock = keyLock.writeLock();
1378                lock.lock();
1379                objectDeque = poolMap.get(k);
1380                if (objectDeque == null) {
1381                    objectDeque = new ObjectDeque<>(fairness);
1382                    objectDeque.getNumInterested().incrementAndGet();
1383                    // NOTE: Keys must always be added to both poolMap and
1384                    //       poolKeyList at the same time while protected by
1385                    //       keyLock.writeLock()
1386                    poolMap.put(k, objectDeque);
1387                    poolKeyList.add(k);
1388                } else {
1389                    objectDeque.getNumInterested().incrementAndGet();
1390                }
1391            } else {
1392                objectDeque.getNumInterested().incrementAndGet();
1393            }
1394        } finally {
1395            lock.unlock();
1396        }
1397        return objectDeque;
1398    }
1399
1400    /**
1401     * Recovers abandoned objects which have been checked out but
1402     * not used since longer than the removeAbandonedTimeout.
1403     *
1404     * @param abandonedConfig The configuration to use to identify abandoned objects
1405     */
1406    @SuppressWarnings("resource") // The PrintWriter is managed elsewhere
1407    private void removeAbandoned(final AbandonedConfig abandonedConfig) {
1408        poolMap.forEach((key, value) -> {
1409            // Generate a list of abandoned objects to remove
1410            final ArrayList<PooledObject<T>> remove = createRemoveList(abandonedConfig, value.getAllObjects());
1411            // Now remove the abandoned objects
1412            remove.forEach(pooledObject -> {
1413                if (abandonedConfig.getLogAbandoned()) {
1414                    pooledObject.printStackTrace(abandonedConfig.getLogWriter());
1415                }
1416                try {
1417                    invalidateObject(key, pooledObject.getObject(), DestroyMode.ABANDONED);
1418                } catch (final Exception e) {
1419                    swallowException(e);
1420                }
1421            });
1422        });
1423    }
1424
1425    /**
1426     * Returns an object to a keyed sub-pool.
1427     * <p>
1428     * If {@link #getMaxIdlePerKey() maxIdle} is set to a positive value and the
1429     * number of idle instances under the given key has reached this value, the
1430     * returning instance is destroyed.
1431     * </p>
1432     * <p>
1433     * If {@link #getTestOnReturn() testOnReturn} == true, the returning
1434     * instance is validated before being returned to the idle instance sub-pool
1435     * under the given key. In this case, if validation fails, the instance is
1436     * destroyed.
1437     * </p>
1438     * <p>
1439     * Exceptions encountered destroying objects for any reason are swallowed
1440     * but notified via a {@link SwallowedExceptionListener}.
1441     * </p>
1442     *
1443     * @param key pool key
1444     * @param obj instance to return to the keyed pool
1445     *
1446     * @throws IllegalStateException if an object is returned to the pool that
1447     *                               was not borrowed from it or if an object is
1448     *                               returned to the pool multiple times
1449     */
1450    @Override
1451    public void returnObject(final K key, final T obj) {
1452
1453        final ObjectDeque<T> objectDeque = poolMap.get(key);
1454
1455        if (objectDeque == null) {
1456            throw new IllegalStateException("No keyed pool found under the given key.");
1457        }
1458
1459        final PooledObject<T> p = objectDeque.getAllObjects().get(new IdentityWrapper<>(obj));
1460
1461        if (p == null) {
1462            throw new IllegalStateException("Returned object not currently part of this pool");
1463        }
1464
1465        markReturningState(p);
1466
1467        final Duration activeTime = p.getActiveDuration();
1468
1469        try {
1470            if (getTestOnReturn() && !factory.validateObject(key, p)) {
1471                try {
1472                    destroy(key, p, true, DestroyMode.NORMAL);
1473                } catch (final Exception e) {
1474                    swallowException(e);
1475                }
1476                whenWaitersAddObject(key, objectDeque.idleObjects);
1477                return;
1478            }
1479
1480            try {
1481                factory.passivateObject(key, p);
1482            } catch (final Exception e1) {
1483                swallowException(e1);
1484                try {
1485                    destroy(key, p, true, DestroyMode.NORMAL);
1486                } catch (final Exception e) {
1487                    swallowException(e);
1488                }
1489                whenWaitersAddObject(key, objectDeque.idleObjects);
1490                return;
1491            }
1492
1493            if (!p.deallocate()) {
1494                throw new IllegalStateException("Object has already been returned to this pool");
1495            }
1496
1497            final int maxIdle = getMaxIdlePerKey();
1498            final LinkedBlockingDeque<PooledObject<T>> idleObjects =
1499                    objectDeque.getIdleObjects();
1500
1501            if (isClosed() || maxIdle > -1 && maxIdle <= idleObjects.size()) {
1502                try {
1503                    destroy(key, p, true, DestroyMode.NORMAL);
1504                } catch (final Exception e) {
1505                    swallowException(e);
1506                }
1507            } else {
1508                if (getLifo()) {
1509                    idleObjects.addFirst(p);
1510                } else {
1511                    idleObjects.addLast(p);
1512                }
1513                if (isClosed()) {
1514                    // Pool closed while object was being added to idle objects.
1515                    // Make sure the returned object is destroyed rather than left
1516                    // in the idle object pool (which would effectively be a leak)
1517                    clear(key);
1518                }
1519            }
1520        } finally {
1521            if (hasBorrowWaiters()) {
1522                reuseCapacity();
1523            }
1524            updateStatsReturn(activeTime);
1525        }
1526    }
1527
1528    /**
1529     * Attempt to create one new instance to serve from the most heavily
1530     * loaded pool that can add a new instance.
1531     *
1532     * This method exists to ensure liveness in the pool when threads are
1533     * parked waiting and capacity to create instances under the requested keys
1534     * subsequently becomes available.
1535     *
1536     * This method is not guaranteed to create an instance and its selection
1537     * of the most loaded pool that can create an instance may not always be
1538     * correct, since it does not lock the pool and instances may be created,
1539     * borrowed, returned or destroyed by other threads while it is executing.
1540     */
1541    private void reuseCapacity() {
1542        final int maxTotalPerKeySave = getMaxTotalPerKey();
1543
1544        // Find the most loaded pool that could take a new instance
1545        int maxQueueLength = 0;
1546        LinkedBlockingDeque<PooledObject<T>> mostLoaded = null;
1547        K loadedKey = null;
1548        for (final Entry<K, GenericKeyedObjectPool<K, T>.ObjectDeque<T>> entry : poolMap.entrySet()) {
1549            final K k = entry.getKey();
1550            final ObjectDeque<T> deque = entry.getValue();
1551            if (deque != null) {
1552                final LinkedBlockingDeque<PooledObject<T>> pool = deque.getIdleObjects();
1553                final int queueLength = pool.getTakeQueueLength();
1554                if (getNumActive(k) < maxTotalPerKeySave && queueLength > maxQueueLength) {
1555                    maxQueueLength = queueLength;
1556                    mostLoaded = pool;
1557                    loadedKey = k;
1558                }
1559            }
1560        }
1561
1562        // Attempt to add an instance to the most loaded pool
1563        if (mostLoaded != null) {
1564            register(loadedKey);
1565            try {
1566                final PooledObject<T> p = create(loadedKey);
1567                if (p != null) {
1568                    addIdleObject(loadedKey, p);
1569                }
1570            } catch (final Exception e) {
1571                swallowException(e);
1572            } finally {
1573                deregister(loadedKey);
1574            }
1575        }
1576    }
1577
1578    /**
1579     * Sets the configuration.
1580     *
1581     * @param conf the new configuration to use. This is used by value.
1582     *
1583     * @see GenericKeyedObjectPoolConfig
1584     */
1585    public void setConfig(final GenericKeyedObjectPoolConfig<T> conf) {
1586        super.setConfig(conf);
1587        setMaxIdlePerKey(conf.getMaxIdlePerKey());
1588        setMaxTotalPerKey(conf.getMaxTotalPerKey());
1589        setMaxTotal(conf.getMaxTotal());
1590        setMinIdlePerKey(conf.getMinIdlePerKey());
1591    }
1592
1593    /**
1594     * Sets the cap on the number of "idle" instances per key in the pool.
1595     * If maxIdlePerKey is set too low on heavily loaded systems it is possible
1596     * you will see objects being destroyed and almost immediately new objects
1597     * being created. This is a result of the active threads momentarily
1598     * returning objects faster than they are requesting them, causing the
1599     * number of idle objects to rise above maxIdlePerKey. The best value for
1600     * maxIdlePerKey for heavily loaded system will vary but the default is a
1601     * good starting point.
1602     *
1603     * @param maxIdlePerKey the maximum number of "idle" instances that can be
1604     *                      held in a given keyed sub-pool. Use a negative value
1605     *                      for no limit
1606     *
1607     * @see #getMaxIdlePerKey
1608     */
1609    public void setMaxIdlePerKey(final int maxIdlePerKey) {
1610        this.maxIdlePerKey = maxIdlePerKey;
1611    }
1612
1613    /**
1614     * Sets the limit on the number of object instances allocated by the pool
1615     * (checked out or idle), per key. When the limit is reached, the sub-pool
1616     * is said to be exhausted. A negative value indicates no limit.
1617     *
1618     * @param maxTotalPerKey the limit on the number of active instances per key
1619     *
1620     * @see #getMaxTotalPerKey
1621     */
1622    public void setMaxTotalPerKey(final int maxTotalPerKey) {
1623        this.maxTotalPerKey = maxTotalPerKey;
1624    }
1625
1626    /**
1627     * Sets the target for the minimum number of idle objects to maintain in
1628     * each of the keyed sub-pools. This setting only has an effect if it is
1629     * positive and {@link #getDurationBetweenEvictionRuns()} is greater than
1630     * zero. If this is the case, an attempt is made to ensure that each
1631     * sub-pool has the required minimum number of instances during idle object
1632     * eviction runs.
1633     * <p>
1634     * If the configured value of minIdlePerKey is greater than the configured
1635     * value for maxIdlePerKey then the value of maxIdlePerKey will be used
1636     * instead.
1637     * </p>
1638     *
1639     * @param minIdlePerKey The minimum size of the each keyed pool
1640     *
1641     * @see #getMinIdlePerKey()
1642     * @see #getMaxIdlePerKey()
1643     * @see #setTimeBetweenEvictionRuns(Duration)
1644     */
1645    public void setMinIdlePerKey(final int minIdlePerKey) {
1646        this.minIdlePerKey = minIdlePerKey;
1647    }
1648
1649    @Override
1650    protected void toStringAppendFields(final StringBuilder builder) {
1651        super.toStringAppendFields(builder);
1652        builder.append(", maxIdlePerKey=");
1653        builder.append(maxIdlePerKey);
1654        builder.append(", minIdlePerKey=");
1655        builder.append(minIdlePerKey);
1656        builder.append(", maxTotalPerKey=");
1657        builder.append(maxTotalPerKey);
1658        builder.append(", factory=");
1659        builder.append(factory);
1660        builder.append(", fairness=");
1661        builder.append(fairness);
1662        builder.append(", poolMap=");
1663        builder.append(poolMap);
1664        builder.append(", poolKeyList=");
1665        builder.append(poolKeyList);
1666        builder.append(", keyLock=");
1667        builder.append(keyLock);
1668        builder.append(", numTotal=");
1669        builder.append(numTotal);
1670        builder.append(", evictionKeyIterator=");
1671        builder.append(evictionKeyIterator);
1672        builder.append(", evictionKey=");
1673        builder.append(evictionKey);
1674        builder.append(", abandonedConfig=");
1675        builder.append(abandonedConfig);
1676    }
1677
1678    /**
1679     * Whether there is at least one thread waiting on this deque, add an pool object.
1680     * @param key pool key.
1681     * @param idleObjects list of idle pool objects.
1682     */
1683    private void whenWaitersAddObject(final K key, final LinkedBlockingDeque<PooledObject<T>> idleObjects) {
1684        if (idleObjects.hasTakeWaiters()) {
1685            try {
1686                addObject(key);
1687            } catch (final Exception e) {
1688                swallowException(e);
1689            }
1690        }
1691    }
1692
1693    /**
1694     * @since 2.10.0
1695     */
1696    @Override
1697    public void use(final T pooledObject) {
1698        final AbandonedConfig abandonedCfg = this.abandonedConfig;
1699        if (abandonedCfg != null && abandonedCfg.getUseUsageTracking()) {
1700            poolMap.values().stream()
1701                .map(pool -> pool.getAllObjects().get(new IdentityWrapper<>(pooledObject)))
1702                .filter(Objects::nonNull)
1703                .findFirst()
1704                .ifPresent(PooledObject::use);
1705        }
1706    }
1707
1708}