需求:redis作为一个内存型的数据库,我们需要对过期key保持关注,从info keyspace中可以看出有多少key没有设置过期时间,那么到底是哪些呢?
说明:关于redis ttl 的返回值,请参考http://redisdoc.com/key/ttl.html
测试数据:
5.5.5.101:6379> get c_100
"100"
5.5.5.101:6379> ttl c_100
(integer) -1
5.5.5.101:6379> expire c_100 600
(integer) 1
5.5.5.101:6379> expire c_1000 600
(integer) 1
5.5.5.101:6379> expire c_888 600
(integer) 1
5.5.5.101:6379> dbsize
(integer) 10000
5.5.5.101:6379> info keyspace
# Keyspace
db0:keys=10000,expires=3,avg_ttl=583699
获取没有设置ttl过期的key名字
db_ip=5.5.5.101
db_port=6379
password=abc123
cursor=0
cnt=100
new_cursor=0
redis-cli -h $db_ip -p $db_port -a $password scan $cursor count $cnt > scan_tmp_result
new_cursor=`sed -n '1p' scan_tmp_result`
sed -n '2,$p' scan_tmp_result > scan_result
cat scan_result |while read line
do
ttl_result=`redis-cli -h $db_ip -p $db_port -a $password ttl $line`
if [[ $ttl_result == -1 ]];then
echo $line >> no_ttl.log
fi
done
while [ $cursor -ne $new_cursor ]
do
redis-cli -h $db_ip -p $db_port -a $password scan $new_cursor count $cnt > scan_tmp_result
new_cursor=`sed -n '1p' scan_tmp_result`
sed -n '2,$p' scan_tmp_result > scan_result
cat scan_result |while read line
do
ttl_result=`redis-cli -h $db_ip -p $db_port -a $password ttl $line`
if [[ $ttl_result == -1 ]];then
echo $line >> no_ttl.log
fi
done
done
rm -rf scan_tmp_result
rm -rf scan_result
查看结果:
[redis@lxd-vm1 ~]$ wc -l no_ttl.log
9997 no_ttl.log
[redis@lxd-vm1 ~]$
线上redis有一些历史遗留的未设置过期时间的key,导致redis空间占用较多,DBA告警后要我们自己清除,于是我写了一个脚本在不影响线上服务的情况下清除(使用keys命令会导致请求hang住)
import sys
import redis
import os
pool = redis.ConnectionPool(host = "host_name",password = "pass_word",port = 6379)
r = redis.StrictRedis(connection_pool = pool)
match = sys.argv[1]+"*"
print(match)
count = 10000
for key in r.scan_iter(match = match,count = count):
time = r.ttl(key)
if time==-1:
r.expire(key,1000)
print("set expire key:",key)
print(time)
这里host_name和pass_word打个码
sys.argv[1]指传入的第一个参数,执行的时候就用python3 xxx.py hello 就会自动扫描hello开头的key,步进10000个,然后将其过期时间设为1000秒,过一会就自动过期了。
在业务低峰期的话步进可以设大点,这样清的会比较快,我们为了清几十万个key花了半个多小时。
ServiceStack.Redis\tests\ServiceStack.Redis.Tests\RedisClientTestsBase.cs
using NUnit.Framework;
using System.Diagnostics;
using System.Text;
namespace ServiceStack.Redis.Tests
{
public class RedisClientTestsBase
{
protected string CleanMask = null;
protected RedisClient Redis;
protected void Log(string fmt, params object[] args)
{
Debug.WriteLine("{0}", string.Format(fmt, args).Trim());
}
[OneTimeSetUp]
public virtual void OnBeforeTestFixture()
{
RedisClient.NewFactoryFn = () => new RedisClient(TestConfig.SingleHost);
using (var redis = RedisClient.New())
{
redis.FlushAll();
}
}
[OneTimeTearDown]
public virtual void OnAfterTestFixture()
{
}
[SetUp]
public virtual void OnBeforeEachTest()
{
Redis = RedisClient.New();
}
[TearDown]
public virtual void OnAfterEachTest()
{
try
{
if (Redis.NamespacePrefix != null && CleanMask == null) CleanMask = Redis.NamespacePrefix + "*";
if (CleanMask != null) Redis.SearchKeys(CleanMask).ForEach(t => Redis.Del(t));
Redis.Dispose();
}
catch (RedisResponseException e)
{
// if exception has that message then it still proves that BgSave works as expected.
if (e.Message.StartsWith("Can't BGSAVE while AOF log rewriting is in progress"))
return;
throw;
}
}
protected string PrefixedKey(string key)
{
return string.Concat(Redis.NamespacePrefix, key);
}
public RedisClient GetRedisClient()
{
var client = new RedisClient(TestConfig.SingleHost);
return client;
}
public RedisClient CreateRedisClient()
{
var client = new RedisClient(TestConfig.SingleHost);
return client;
}
public string GetString(byte[] stringBytes)
{
return Encoding.UTF8.GetString(stringBytes);
}
public byte[] GetBytes(string stringValue)
{
return Encoding.UTF8.GetBytes(stringValue);
}
}
}
ServiceStack.Redis\tests\ServiceStack.Redis.Tests\RedisScanTests.cs
[Test]
public void Can_scan_100_collection_over_cursor()
{
var allKeys = new HashSet<string>();
Redis.FlushAll();
var keys = 100.Times(x => "KEY" + x);
Redis.SetAll(keys.ToSafeDictionary(x => x));
var i = 0;
var ret = new ScanResult();
while (true)
{
ret = Redis.Scan(ret.Cursor, 10);
i++;
ret.AsStrings().ForEach(x => allKeys.Add(x));
if (ret.Cursor == 0) break;
}
Assert.That(i, Is.GreaterThanOrEqualTo(2));
Assert.That(allKeys.Count, Is.EqualTo(keys.Count));
Assert.That(allKeys, Is.EquivalentTo(keys));
}