需求:redis作为一个内存型的数据库,我们需要对过期key保持关注,从info keyspace中可以看出有多少key没有设置过期时间,那么到底是哪些呢?
说明:关于redis ttl 的返回值,请参考http://redisdoc.com/key/ttl.html
测试数据:
5.5.5.101:6379> get c_100"100"5.5.5.101:6379> ttl c_100(integer) -15.5.5.101:6379> expire c_100 600(integer) 15.5.5.101:6379> expire c_1000 600(integer) 15.5.5.101:6379> expire c_888 600(integer) 15.5.5.101:6379> dbsize(integer) 100005.5.5.101:6379> info keyspace# Keyspacedb0:keys=10000,expires=3,avg_ttl=583699
获取没有设置ttl过期的key名字
db_ip=5.5.5.101db_port=6379password=abc123cursor=0cnt=100new_cursor=0redis-cli -h $db_ip -p $db_port -a $password scan $cursor count $cnt > scan_tmp_resultnew_cursor=`sed -n '1p' scan_tmp_result`sed -n '2,$p' scan_tmp_result > scan_resultcat scan_result |while read linedottl_result=`redis-cli -h $db_ip -p $db_port -a $password ttl $line`if [[ $ttl_result == -1 ]];thenecho $line >> no_ttl.logfidonewhile [ $cursor -ne $new_cursor ]doredis-cli -h $db_ip -p $db_port -a $password scan $new_cursor count $cnt > scan_tmp_resultnew_cursor=`sed -n '1p' scan_tmp_result`sed -n '2,$p' scan_tmp_result > scan_resultcat scan_result |while read linedottl_result=`redis-cli -h $db_ip -p $db_port -a $password ttl $line`if [[ $ttl_result == -1 ]];thenecho $line >> no_ttl.logfidonedonerm -rf scan_tmp_resultrm -rf scan_result
查看结果:
[redis@lxd-vm1 ~]$ wc -l no_ttl.log9997 no_ttl.log[redis@lxd-vm1 ~]$
线上redis有一些历史遗留的未设置过期时间的key,导致redis空间占用较多,DBA告警后要我们自己清除,于是我写了一个脚本在不影响线上服务的情况下清除(使用keys命令会导致请求hang住)
import sysimport redisimport ospool = redis.ConnectionPool(host = "host_name",password = "pass_word",port = 6379)r = redis.StrictRedis(connection_pool = pool)match = sys.argv[1]+"*"print(match)count = 10000for key in r.scan_iter(match = match,count = count):time = r.ttl(key)if time==-1:r.expire(key,1000)print("set expire key:",key)print(time)
这里host_name和pass_word打个码
sys.argv[1]指传入的第一个参数,执行的时候就用python3 xxx.py hello 就会自动扫描hello开头的key,步进10000个,然后将其过期时间设为1000秒,过一会就自动过期了。
在业务低峰期的话步进可以设大点,这样清的会比较快,我们为了清几十万个key花了半个多小时。
ServiceStack.Redis\tests\ServiceStack.Redis.Tests\RedisClientTestsBase.csusing NUnit.Framework;using System.Diagnostics;using System.Text;namespace ServiceStack.Redis.Tests{public class RedisClientTestsBase{protected string CleanMask = null;protected RedisClient Redis;protected void Log(string fmt, params object[] args){Debug.WriteLine("{0}", string.Format(fmt, args).Trim());}[OneTimeSetUp]public virtual void OnBeforeTestFixture(){RedisClient.NewFactoryFn = () => new RedisClient(TestConfig.SingleHost);using (var redis = RedisClient.New()){redis.FlushAll();}}[OneTimeTearDown]public virtual void OnAfterTestFixture(){}[SetUp]public virtual void OnBeforeEachTest(){Redis = RedisClient.New();}[TearDown]public virtual void OnAfterEachTest(){try{if (Redis.NamespacePrefix != null && CleanMask == null) CleanMask = Redis.NamespacePrefix + "*";if (CleanMask != null) Redis.SearchKeys(CleanMask).ForEach(t => Redis.Del(t));Redis.Dispose();}catch (RedisResponseException e){// if exception has that message then it still proves that BgSave works as expected.if (e.Message.StartsWith("Can't BGSAVE while AOF log rewriting is in progress"))return;throw;}}protected string PrefixedKey(string key){return string.Concat(Redis.NamespacePrefix, key);}public RedisClient GetRedisClient(){var client = new RedisClient(TestConfig.SingleHost);return client;}public RedisClient CreateRedisClient(){var client = new RedisClient(TestConfig.SingleHost);return client;}public string GetString(byte[] stringBytes){return Encoding.UTF8.GetString(stringBytes);}public byte[] GetBytes(string stringValue){return Encoding.UTF8.GetBytes(stringValue);}}}ServiceStack.Redis\tests\ServiceStack.Redis.Tests\RedisScanTests.cs[Test]public void Can_scan_100_collection_over_cursor(){var allKeys = new HashSet<string>();Redis.FlushAll();var keys = 100.Times(x => "KEY" + x);Redis.SetAll(keys.ToSafeDictionary(x => x));var i = 0;var ret = new ScanResult();while (true){ret = Redis.Scan(ret.Cursor, 10);i++;ret.AsStrings().ForEach(x => allKeys.Add(x));if (ret.Cursor == 0) break;}Assert.That(i, Is.GreaterThanOrEqualTo(2));Assert.That(allKeys.Count, Is.EqualTo(keys.Count));Assert.That(allKeys, Is.EquivalentTo(keys));}
